Restored --download, although not done yet

The problem at this point is that it's recording prices in the price
database multiple times; it should only need to download a price for
each commodity once per day.

8 files changed, 164 insertions, 83 deletions

diff --git a/src/commodity.cc b/src/commodity.cc
index 095912dd..81c9b7e6 100644
--- a/src/commodity.cc
+++ b/src/commodity.cc
@@ -36,6 +36,10 @@
 
 namespace ledger {
 
+optional<path> commodity_t::price_db;
+long           commodity_t::download_leeway = 86400;
+bool           commodity_t::download_quotes;
+
 void commodity_t::base_t::history_t::add_price(commodity_t&      source,
 					       const datetime_t& date,
 					       const amount_t&	 price,
@@ -105,10 +109,108 @@ bool commodity_t::base_t::varied_history_t::remove_price(const datetime_t&  date
   return false;
 }
 
+optional<price_point_t> commodity_t::parse_commodity_price(char * line)
+{
+  char * date_field_ptr = line;
+  char * time_field_ptr = next_element(date_field_ptr);
+  if (! time_field_ptr) return none;
+  string date_field = date_field_ptr;
+
+  char *     symbol_and_price;
+  datetime_t datetime;
+
+  if (std::isdigit(time_field_ptr[0])) {
+    symbol_and_price = next_element(time_field_ptr);
+    if (! symbol_and_price) return none;
+    datetime = parse_datetime(date_field + " " + time_field_ptr);
+  } else {
+    symbol_and_price = time_field_ptr;
+    datetime = parse_datetime(date_field);
+  }
+
+  string symbol;
+  parse_symbol(symbol_and_price, symbol);
+
+  price_point_t point;
+  point.when = datetime;
+  point.price.parse(symbol_and_price);
+  VERIFY(point.price.valid());
+
+  if (commodity_t * commodity =
+      amount_t::current_pool->find_or_create(symbol)) {
+    commodity->add_price(point.when, point.price, true);
+    commodity->add_flags(COMMODITY_KNOWN);
+    return point;
+  }
+
+  return none;
+}
+
+
+optional<price_point_t>
+commodity_t::download_quote(const optional<commodity_t&>& commodity) const
+{
+  DEBUG("commodity.download", "downloading quote for symbol " << symbol());
+#if defined(DEBUG_ON)
+  if (commodity)
+    DEBUG("commodity.download",
+	  "  in terms of commodity " << commodity->symbol());
+#endif
+
+  char buf[256];
+  buf[0] = '\0';
+
+  string getquote_cmd("getquote \"");
+  getquote_cmd += symbol();
+  getquote_cmd += "\" \"";
+  if (commodity)
+    getquote_cmd += commodity->symbol();
+  getquote_cmd += "\"";
+
+  DEBUG("commodity.download", "invoking command: " << getquote_cmd);
+
+  bool success = true;
+  if (FILE * fp = popen(getquote_cmd.c_str(), "r")) {
+    if (std::feof(fp) || ! std::fgets(buf, 255, fp))
+      success = false;
+    if (pclose(fp) != 0)
+      success = false;
+  } else {
+    success = false;
+  }
+
+  if (success && buf[0]) {
+    char * p = std::strchr(buf, '\n');
+    if (p) *p = '\0';
+
+    DEBUG("commodity.download", "downloaded quote: " << buf);
+
+    optional<price_point_t> point = parse_commodity_price(buf);
+
+    if (point) {
+      if (price_db) {
+#if defined(__GNUG__) && __GNUG__ < 3
+	ofstream database(*price_db, ios::out | ios::app);
+#else
+	ofstream database(*price_db, std::ios_base::out | std::ios_base::app);
+#endif
+	database << "P " << format_datetime(point->when, string("%Y/%m/%d %H:%M:%S"))
+		 << " " << symbol() << " " << point->price << std::endl;
+      }
+      return point;
+    }
+  } else {
+    throw_(std::runtime_error,
+	   _("Failed to download price for '%1' (command: \"getquote %2\")")
+	   << symbol() << symbol());
+  }
+  return none;
+}
+
 optional<price_point_t>
   commodity_t::base_t::history_t::
-    find_price(const optional<datetime_t>&   moment,
-	       const optional<datetime_t>&   oldest
+    find_price(const optional<datetime_t>& moment,
+	       const optional<datetime_t>& oldest
 #if defined(DEBUG_ON)
 	       , const int indent
 #endif
@@ -185,16 +287,6 @@ optional<price_point_t>
     }
   }
 
-#if 0
-  if (! has_flags(COMMODITY_NOMARKET) && parent().get_quote) {
-    if (optional<amount_t> quote = parent().get_quote
-	(*this, age, moment,
-	 (hist && hist->prices.size() > 0 ?
-	  (*hist->prices.rbegin()).first : optional<datetime_t>())))
-      return *quote;
-  }
-#endif
-
   if (! found) {
 #if defined(DEBUG_ON)
     DEBUG_INDENT("commodity.prices.find", indent);
@@ -351,7 +443,29 @@ optional<price_point_t>
     DEBUG_INDENT("commodity.prices.find", indent);
     DEBUG("commodity.prices.find",
 	  "  found price " << best.price << " from " << best.when);
+    DEBUG("commodity.download",
+	  "found price " << best.price << " from " << best.when);
+    if (moment)
+      DEBUG("commodity.download", "moment = " << *moment);
+    DEBUG("commodity.download", "leeway = " << download_leeway);
+    if (moment)
+      DEBUG("commodity.download",
+	    "slip.moment = " << (*moment - best.when).total_seconds());
+    else
+      DEBUG("commodity.download",
+	    "slip.now = " << (CURRENT_TIME() - best.when).total_seconds());
 #endif
+    if (download_quotes &&
+	! source.has_flags(COMMODITY_NOMARKET) &&
+	((! moment &&
+	  (CURRENT_TIME() - best.when).total_seconds() > download_leeway) ||
+	 (moment &&
+	  (*moment - best.when).total_seconds() > download_leeway))) {
+      DEBUG("commodity.download",
+	    "attempting to download a more current quote...");
+      if (optional<price_point_t> quote = source.download_quote(commodity))
+	return quote;
+    }
     return best;
   }
   return none;
@@ -368,7 +482,8 @@ optional<commodity_t::base_t::history_t&>
 #if 0
       // jww (2008-09-20): Document which option switch to use here
       throw_(commodity_error,
-	     _("Cannot determine price history: prices known for multiple commodities (use -x)"));
+	     _("Cannot determine price history: "
+	       "prices known for multiple commodities (use -x)"));
 #endif
     comm = (*histories.begin()).first;
   } else {
diff --git a/src/commodity.h b/src/commodity.h
index c15a32f0..c678293e 100644
--- a/src/commodity.h
+++ b/src/commodity.h
@@ -86,7 +86,6 @@ public:
     struct history_t
     {
       history_map prices;
-      ptime	  last_lookup;
 
       void add_price(commodity_t&	source,
 		     const datetime_t&	date,
@@ -332,6 +331,15 @@ public:
   // Methods related to parsing, reading, writing, etc., the commodity
   // itself.
 
+  static optional<path> price_db;
+  static long           download_leeway;
+  static bool           download_quotes;
+
+  static optional<price_point_t> parse_commodity_price(char * line);
+
+  optional<price_point_t>
+  download_quote(const optional<commodity_t&>& commodity = none) const;
+
   static void parse_symbol(std::istream& in, string& symbol);
   static void parse_symbol(char *& p, string& symbol);
   static string parse_symbol(std::istream& in) {
diff --git a/src/global.cc b/src/global.cc
index e8c6f17d..5d5e5836 100644
--- a/src/global.cc
+++ b/src/global.cc
@@ -415,10 +415,21 @@ void global_scope_t::normalize_report_options(const string& verb)
 
   report_t& rep(report());
 
-  // jww (2009-02-09): These global are a hack, but hard to avoid.
+  // jww (2009-02-09): These globals are a hack, but hard to avoid.
   item_t::use_effective_date		= rep.HANDLED(effective);
   rep.session.commodity_pool->keep_base = rep.HANDLED(base);
 
+  commodity_t::download_quotes = rep.session.HANDLED(download);
+
+  if (rep.session.HANDLED(price_exp_))
+    commodity_t::download_leeway =
+      rep.session.HANDLER(price_exp_).value.as_long();
+
+  if (rep.session.HANDLED(price_db_))
+    commodity_t::price_db = rep.session.HANDLER(price_db_).str();
+  else
+    commodity_t::price_db = none;
+
   if (rep.HANDLED(date_format_)) {
     output_datetime_format = rep.HANDLER(date_format_).str() + " %H:%M:%S";
     output_date_format     = rep.HANDLER(date_format_).str();
diff --git a/src/report.cc b/src/report.cc
index 5d1d0da8..d3c936f1 100644
--- a/src/report.cc
+++ b/src/report.cc
@@ -474,9 +474,6 @@ option_t<report_t> * report_t::lookup_option(const char * p)
   case 'Y':
     OPT_CH(yearly);
     break;
-  case 'Z':
-    OPT_CH(price_exp_);
-    break;
   case 'a':
     OPT(abbrev_len_);
     else OPT(account_);
@@ -557,7 +554,6 @@ option_t<report_t> * report_t::lookup_option(const char * p)
     else OPT(lots);
     else OPT(lots_actual);
     else OPT_ALT(tail_, last_);
-    else OPT_ALT(price_exp_, leeway_);
     break;
   case 'm':
     OPT(market);
@@ -582,7 +578,6 @@ option_t<report_t> * report_t::lookup_option(const char * p)
     else OPT(plot_amount_format_);
     else OPT(plot_total_format_);
     else OPT(price);
-    else OPT(price_exp_);
     else OPT(prices_format_);
     else OPT(pricesdb_format_);
     else OPT(print_format_);
diff --git a/src/report.h b/src/report.h
index 63a1a045..147620f8 100644
--- a/src/report.h
+++ b/src/report.h
@@ -255,7 +255,6 @@ public:
     HANDLER(plot_amount_format_).report(out);
     HANDLER(plot_total_format_).report(out);
     HANDLER(price).report(out);
-    HANDLER(price_exp_).report(out);
     HANDLER(prices_format_).report(out);
     HANDLER(pricesdb_format_).report(out);
     HANDLER(print_format_).report(out);
@@ -619,8 +618,6 @@ public:
       parent->HANDLER(amount_).set_expr(string("--price"), "price");
     });
 
-  OPTION(report_t, price_exp_); // -Z
-
   OPTION__(report_t, prices_format_, CTOR(report_t, prices_format_) {
       on(none,
 	 "%-.9(date) %-8(account) %(justify(scrub(display_amount), 12, "
diff --git a/src/session.cc b/src/session.cc
index 6b891e1b..d5cc3b38 100644
--- a/src/session.cc
+++ b/src/session.cc
@@ -219,6 +219,12 @@ void session_t::clean_accounts()
 option_t<session_t> * session_t::lookup_option(const char * p)
 {
   switch (*p) {
+  case 'Q':
+    OPT_CH(download); // -Q
+    break;
+  case 'Z':
+    OPT_CH(price_exp_);
+    break;
   case 'a':
     OPT_(account_); // -a
     break;
@@ -232,17 +238,15 @@ option_t<session_t> * session_t::lookup_option(const char * p)
     OPT(input_date_format_);
     break;
   case 'l':
-    OPT(leeway_);
+    OPT_ALT(price_exp_, leeway_);
     break;
   case 'p':
     OPT(price_db_);
+    else OPT(price_exp_);
     break;
   case 's':
     OPT(strict);
     break;
-  case 'Q':
-    OPT_CH(download); // -Q
-    break;
   }
   return NULL;
 }
diff --git a/src/session.h b/src/session.h
index fc6bf4f3..5d6a12b9 100644
--- a/src/session.h
+++ b/src/session.h
@@ -105,10 +105,10 @@ public:
   {
     HANDLER(account_).report(out);
     HANDLER(download).report(out);
-    HANDLER(leeway_).report(out);
     HANDLER(file_).report(out);
     HANDLER(input_date_format_).report(out);
     HANDLER(price_db_).report(out);
+    HANDLER(price_exp_).report(out);
     HANDLER(strict).report(out);
   }
 
@@ -124,8 +124,8 @@ public:
   OPTION(session_t, download); // -Q
 
   OPTION__
-  (session_t, leeway_,
-   CTOR(session_t, leeway_) { value = 24L * 3600L; }
+  (session_t, price_exp_, // -Z
+   CTOR(session_t, price_exp_) { value = 24L * 3600L; }
    DO_(args) {
      value = args[1].to_long() * 60L;
    });
diff --git a/src/textual.cc b/src/textual.cc
index 3cf3c13c..6f96ba99 100644
--- a/src/textual.cc
+++ b/src/textual.cc
@@ -455,67 +455,18 @@ void instance_t::price_conversion_directive(char * line)
   }
 }
 
-namespace {
-  void parse_symbol(char *& p, string& symbol)
-  {
-    if (*p == '"') {
-      char * q = std::strchr(p + 1, '"');
-      if (! q)
-	throw parse_error(_("Quoted commodity symbol lacks closing quote"));
-      symbol = string(p + 1, 0, q - p - 1);
-      p = q + 2;
-    } else {
-      char * q = next_element(p);
-      symbol = p;
-      if (q)
-	p = q;
-      else
-	p += symbol.length();
-    }
-    if (symbol.empty())
-      throw parse_error(_("Failed to parse commodity"));
-  }
-}
-
 void instance_t::price_xact_directive(char * line)
 {
-  char * date_field_ptr = skip_ws(line + 1);
-  char * time_field_ptr = next_element(date_field_ptr);
-  if (! time_field_ptr) return;
-  string date_field = date_field_ptr;
-
-  char *     symbol_and_price;
-  datetime_t datetime;
-
-  if (std::isdigit(time_field_ptr[0])) {
-    symbol_and_price = next_element(time_field_ptr);
-    if (! symbol_and_price) return;
-    datetime = parse_datetime(date_field + " " + time_field_ptr,
-			      current_year);
-  } else {
-    symbol_and_price = time_field_ptr;
-    datetime = parse_datetime(date_field, current_year);
-  }
-
-  string symbol;
-  parse_symbol(symbol_and_price, symbol);
-  amount_t price(symbol_and_price);
-  VERIFY(price.valid());
-
-  if (commodity_t * commodity =
-      amount_t::current_pool->find_or_create(symbol)) {
-    commodity->add_price(datetime, price, true);
-    commodity->add_flags(COMMODITY_KNOWN);
-  } else {
-    assert(false);
-  }
+  optional<price_point_t> point =
+    commodity_t::parse_commodity_price(skip_ws(line + 1));
+  assert(point);
 }
 
 void instance_t::nomarket_directive(char * line)
 {
   char * p = skip_ws(line + 1);
   string symbol;
-  parse_symbol(p, symbol);
+  commodity_t::parse_symbol(p, symbol);
 
   if (commodity_t * commodity =
       amount_t::current_pool->find_or_create(symbol))