Update to Org 9.4.1

1 files changed, 119 insertions, 100 deletions

diff --git a/lisp/org/ob-python.el b/lisp/org/ob-python.el
index 823f6e63d57..ffb8ee855ef 100644
--- a/lisp/org/ob-python.el
+++ b/lisp/org/ob-python.el
@@ -4,6 +4,7 @@
 
 ;; Authors: Eric Schulte
 ;;	 Dan Davison
+;; Maintainer: Jack Kamm <jackkamm@gmail.com>
 ;; Keywords: literate programming, reproducible research
 ;; Homepage: https://orgmode.org
 
@@ -29,10 +30,11 @@
 ;;; Code:
 (require 'ob)
 (require 'org-macs)
+(require 'python)
 
-(declare-function py-shell "ext:python-mode" (&optional argprompt))
+(declare-function py-shell "ext:python-mode" (&rest args))
 (declare-function py-toggle-shells "ext:python-mode" (arg))
-(declare-function run-python "ext:python" (&optional cmd dedicated show))
+(declare-function py-shell-send-string "ext:python-mode" (strg &optional process))
 
 (defvar org-babel-tangle-lang-exts)
 (add-to-list 'org-babel-tangle-lang-exts '("python" . "py"))
@@ -104,7 +106,8 @@ VARS contains resolved variable references."
     (org-babel-comint-in-buffer session
       (mapc (lambda (var)
               (end-of-line 1) (insert var) (comint-send-input)
-              (org-babel-comint-wait-for-output session)) var-lines))
+              (org-babel-comint-wait-for-output session))
+	    var-lines))
     session))
 
 (defun org-babel-load-session:python (session body params)
@@ -177,42 +180,40 @@ Emacs-lisp table, otherwise return the results as a string."
   "Initiate a python session.
 If there is not a current inferior-process-buffer in SESSION
 then create.  Return the initialized session."
-  (require org-babel-python-mode)
   (save-window-excursion
     (let* ((session (if session (intern session) :default))
-           (python-buffer (org-babel-python-session-buffer session))
+           (py-buffer (org-babel-python-session-buffer session))
 	   (cmd (if (member system-type '(cygwin windows-nt ms-dos))
 		    (concat org-babel-python-command " -i")
 		  org-babel-python-command)))
       (cond
-       ((and (eq 'python org-babel-python-mode)
-	     (fboundp 'run-python)) ; python.el
-	(if (not (version< "24.1" emacs-version))
-	    (run-python cmd)
-	  (unless python-buffer
-	    (setq python-buffer (org-babel-python-with-earmuffs session)))
-	  (let ((python-shell-buffer-name
-		 (org-babel-python-without-earmuffs python-buffer)))
-	    (run-python cmd))))
+       ((eq 'python org-babel-python-mode) ; python.el
+	(unless py-buffer
+	  (setq py-buffer (org-babel-python-with-earmuffs session)))
+	(let ((python-shell-buffer-name
+	       (org-babel-python-without-earmuffs py-buffer)))
+	  (run-python cmd)
+	  (sleep-for 0 10)))
        ((and (eq 'python-mode org-babel-python-mode)
 	     (fboundp 'py-shell)) ; python-mode.el
+	(require 'python-mode)
 	;; Make sure that py-which-bufname is initialized, as otherwise
 	;; it will be overwritten the first time a Python buffer is
 	;; created.
 	(py-toggle-shells py-default-interpreter)
 	;; `py-shell' creates a buffer whose name is the value of
 	;; `py-which-bufname' with '*'s at the beginning and end
-	(let* ((bufname (if (and python-buffer (buffer-live-p python-buffer))
+	(let* ((bufname (if (and py-buffer (buffer-live-p py-buffer))
 			    (replace-regexp-in-string ;; zap surrounding *
-			     "^\\*\\([^*]+\\)\\*$" "\\1" python-buffer)
+			     "^\\*\\([^*]+\\)\\*$" "\\1" py-buffer)
 			  (concat "Python-" (symbol-name session))))
 	       (py-which-bufname bufname))
-	  (py-shell)
-	  (setq python-buffer (org-babel-python-with-earmuffs bufname))))
+	  (setq py-buffer (org-babel-python-with-earmuffs bufname))
+	  (py-shell nil nil t org-babel-python-command py-buffer nil nil t nil)))
        (t
 	(error "No function available for running an inferior Python")))
       (setq org-babel-python-buffers
-	    (cons (cons session python-buffer)
+	    (cons (cons session py-buffer)
 		  (assq-delete-all session org-babel-python-buffers)))
       session)))
 
@@ -222,8 +223,9 @@ then create.  Return the initialized session."
     (org-babel-python-session-buffer
      (org-babel-python-initiate-session-by-key session))))
 
-(defvar org-babel-python-eoe-indicator "'org_babel_python_eoe'"
+(defvar org-babel-python-eoe-indicator "org_babel_python_eoe"
   "A string to indicate that evaluation has completed.")
+
 (defconst org-babel-python-wrapper-method
   "
 def main():
@@ -238,14 +240,39 @@ def main():
 
 open('%s', 'w').write( pprint.pformat(main()) )")
 
-(defconst org-babel-python--exec-tmpfile
-  (concat
-   "__org_babel_python_fname = '%s'; "
-   "__org_babel_python_fh = open(__org_babel_python_fname); "
-   "exec(compile("
-   "__org_babel_python_fh.read(), __org_babel_python_fname, 'exec'"
-   ")); "
-   "__org_babel_python_fh.close()"))
+(defconst org-babel-python--exec-tmpfile "\
+with open('%s') as __org_babel_python_tmpfile:
+    exec(compile(__org_babel_python_tmpfile.read(), __org_babel_python_tmpfile.name, 'exec'))"
+  "Template for Python session command with output results.
+
+Has a single %s escape, the tempfile containing the source code
+to evaluate.")
+
+(defun org-babel-python-format-session-value
+    (src-file result-file result-params)
+  "Return Python code to evaluate SRC-FILE and write result to RESULT-FILE."
+  (format "\
+import ast
+with open('%s') as __org_babel_python_tmpfile:
+    __org_babel_python_ast = ast.parse(__org_babel_python_tmpfile.read())
+__org_babel_python_final = __org_babel_python_ast.body[-1]
+if isinstance(__org_babel_python_final, ast.Expr):
+    __org_babel_python_ast.body = __org_babel_python_ast.body[:-1]
+    exec(compile(__org_babel_python_ast, '<string>', 'exec'))
+    __org_babel_python_final = eval(compile(ast.Expression(
+        __org_babel_python_final.value), '<string>', 'eval'))
+    with open('%s', 'w') as __org_babel_python_tmpfile:
+        if %s:
+            import pprint
+            __org_babel_python_tmpfile.write(pprint.pformat(__org_babel_python_final))
+        else:
+            __org_babel_python_tmpfile.write(str(__org_babel_python_final))
+else:
+    exec(compile(__org_babel_python_ast, '<string>', 'exec'))
+    __org_babel_python_final = None"
+	  (org-babel-process-file-name src-file 'noquote)
+	  (org-babel-process-file-name result-file 'noquote)
+	  (if (member "pp" result-params) "True" "False")))
 
 (defun org-babel-python-evaluate
   (session body &optional result-type result-params preamble)
@@ -256,6 +283,19 @@ open('%s', 'w').write( pprint.pformat(main()) )")
     (org-babel-python-evaluate-external-process
      body result-type result-params preamble)))
 
+(defun org-babel-python--shift-right (body &optional count)
+  (with-temp-buffer
+    (python-mode)
+    (insert body)
+    (goto-char (point-min))
+    (while (not (eobp))
+      (unless (python-syntax-context 'string)
+	(python-indent-shift-right (line-beginning-position)
+				   (line-end-position)
+				   count))
+      (forward-line 1))
+    (buffer-string)))
+
 (defun org-babel-python-evaluate-external-process
     (body &optional result-type result-params preamble)
   "Evaluate BODY in external python process.
@@ -276,89 +316,70 @@ last statement in BODY, as elisp."
 			(if (member "pp" result-params)
 			    org-babel-python-pp-wrapper-method
 			  org-babel-python-wrapper-method)
-			(mapconcat
-			 (lambda (line) (format "\t%s" line))
-			 (split-string (org-remove-indentation (org-trim body))
-				       "[\r\n]")
-			 "\n")
+			(org-babel-python--shift-right body)
 			(org-babel-process-file-name tmp-file 'noquote))))
 		     (org-babel-eval-read-file tmp-file))))))
     (org-babel-result-cond result-params
       raw
       (org-babel-python-table-or-string (org-trim raw)))))
 
+(defun org-babel-python--send-string (session body)
+  "Pass BODY to the Python process in SESSION.
+Return output."
+  (with-current-buffer session
+    (let* ((string-buffer "")
+	   (comint-output-filter-functions
+	    (cons (lambda (text) (setq string-buffer
+				       (concat string-buffer text)))
+		  comint-output-filter-functions))
+	   (body (format "\
+try:
+%s
+except:
+    raise
+finally:
+    print('%s')"
+			 (org-babel-python--shift-right body 4)
+			 org-babel-python-eoe-indicator)))
+      (if (not (eq 'python-mode org-babel-python-mode))
+	  (let ((python-shell-buffer-name
+		 (org-babel-python-without-earmuffs session)))
+	    (python-shell-send-string body))
+	(require 'python-mode)
+	(py-shell-send-string body (get-buffer-process session)))
+      ;; same as `python-shell-comint-end-of-output-p' in emacs-25.1+
+      (while (not (string-match
+		   org-babel-python-eoe-indicator
+		   string-buffer))
+	(accept-process-output (get-buffer-process (current-buffer))))
+      (org-babel-chomp (substring string-buffer 0 (match-beginning 0))))))
+
 (defun org-babel-python-evaluate-session
     (session body &optional result-type result-params)
   "Pass BODY to the Python process in SESSION.
 If RESULT-TYPE equals `output' then return standard output as a
 string.  If RESULT-TYPE equals `value' then return the value of the
 last statement in BODY, as elisp."
-  (let* ((send-wait (lambda () (comint-send-input nil t) (sleep-for 0 5)))
-	 (dump-last-value
-	  (lambda
-	    (tmp-file pp)
-	    (mapc
-	     (lambda (statement) (insert statement) (funcall send-wait))
-	     (if pp
-		 (list
-		  "import pprint"
-		  (format "open('%s', 'w').write(pprint.pformat(_))"
-			  (org-babel-process-file-name tmp-file 'noquote)))
-	       (list (format "open('%s', 'w').write(str(_))"
-			     (org-babel-process-file-name tmp-file
-                                                          'noquote)))))))
-	 (last-indent 0)
-	 (input-body (lambda (body)
-		       (dolist (line (split-string body "[\r\n]"))
-			 ;; Insert a blank line to end an indent
-			 ;; block.
-			 (let ((curr-indent (string-match "\\S-" line)))
-			   (if curr-indent
-			       (progn
-				 (when (< curr-indent last-indent)
-				   (insert "")
-				   (funcall send-wait))
-				 (setq last-indent curr-indent))
-			     (setq last-indent 0)))
-			 (insert line)
-			 (funcall send-wait))
-		       (funcall send-wait)))
+  (let* ((tmp-src-file (org-babel-temp-file "python-"))
          (results
-          (pcase result-type
-            (`output
-	     (let ((body (if (string-match-p ".\n+." body) ; Multiline
-			     (let ((tmp-src-file (org-babel-temp-file
-						  "python-")))
-			       (with-temp-file tmp-src-file (insert body))
-			       (format org-babel-python--exec-tmpfile
-				       tmp-src-file))
-			   body)))
-	       (mapconcat
-		#'org-trim
-		(butlast
-		 (org-babel-comint-with-output
-		     (session org-babel-python-eoe-indicator t body)
-		   (funcall input-body body)
-		   (funcall send-wait) (funcall send-wait)
-		   (insert org-babel-python-eoe-indicator)
-		   (funcall send-wait))
-		 2) "\n")))
-            (`value
-             (let ((tmp-file (org-babel-temp-file "python-")))
-               (org-babel-comint-with-output
-                   (session org-babel-python-eoe-indicator nil body)
-                 (let ((comint-process-echoes nil))
-                   (funcall input-body body)
-                   (funcall dump-last-value tmp-file
-                            (member "pp" result-params))
-                   (funcall send-wait) (funcall send-wait)
-                   (insert org-babel-python-eoe-indicator)
-                   (funcall send-wait)))
-               (org-babel-eval-read-file tmp-file))))))
-    (unless (string= (substring org-babel-python-eoe-indicator 1 -1) results)
-      (org-babel-result-cond result-params
-	results
-        (org-babel-python-table-or-string results)))))
+	  (progn
+	    (with-temp-file tmp-src-file (insert body))
+            (pcase result-type
+	      (`output
+	       (let ((body (format org-babel-python--exec-tmpfile
+				   (org-babel-process-file-name
+				    tmp-src-file 'noquote))))
+		 (org-babel-python--send-string session body)))
+              (`value
+               (let* ((tmp-results-file (org-babel-temp-file "python-"))
+		      (body (org-babel-python-format-session-value
+			     tmp-src-file tmp-results-file result-params)))
+		 (org-babel-python--send-string session body)
+		 (sleep-for 0 10)
+		 (org-babel-eval-read-file tmp-results-file)))))))
+    (org-babel-result-cond result-params
+      results
+      (org-babel-python-table-or-string results))))
 
 (defun org-babel-python-read-string (string)
   "Strip \\='s from around Python string."
@@ -369,6 +390,4 @@ last statement in BODY, as elisp."
 
 (provide 'ob-python)
 
-
-
 ;;; ob-python.el ends here