add variable interpolation

docs/source/usage.md

@@ -67,23 +67,57 @@ You can then use, e.g.,
 
 ### IPython magic
 
-In IPython (and therefore in Jupyter), you can directly execute Julia
-code using `%%julia` magic:
+In IPython (and therefore in Jupyter), you can directly execute Julia code using `%julia` magic:
 
-```
+```python
 In [1]: %load_ext julia.magic
 Initializing Julia interpreter. This may take some time...
 
-In [2]: %%julia
-   ...: Base.banner(IOContext(stdout, :color=>true))
-               _
-   _       _ _(_)_     |  Documentation: https://docs.julialang.org
-  (_)     | (_) (_)    |
-   _ _   _| |_  __ _   |  Type "?" for help, "]?" for Pkg help.
-  | | | | | | |/ _` |  |
-  | | |_| | | | (_| |  |  Version 1.0.1 (2018-09-29)
- _/ |\__'_|_|_|\__'_|  |  Official https://julialang.org/ release
-|__/                   |
+In [2]: %julia [1 2; 3 4] .+ 1 
+Out[2]: 
+array([[2, 3],
+       [4, 5]], dtype=int64)
+```
+
+You can "interpolate" Python objects into Julia code via `$var` for the value of single variables or `py"expression"` for the result of any arbitrary Python code:
+```julia
+In [3]: arr = [1,2,3]
+
+In [4]: %julia $arr .+ 1
+Out[4]: 
+array([2, 3, 4], dtype=int64)
+
+In [5]: %julia sum(py"[x**2 for x in arr]")
+Out[5]: 14
+```
+
+Python interpolation is not performed inside of strings (instead this is treated as regular Julia string interpolation), and can also be overridden elsewhere by using `$$...` to insert a literal `$...`:
+
+```julia
+In [6]: %julia foo=3; "$foo"
+Out[6]: '3'
+
+In [7]: %julia bar=3; :($$bar)
+Out[7]: 3
+```
+
+Variables are automatically converted between equivalent Python/Julia types (should they exist). You can turn this off by appending `o` to the Python string:
+
+```python
+In [8]: %julia typeof(py"1"), typeof(py"1"o)
+Out[8]: (<PyCall.jlwrap Int64>, <PyCall.jlwrap PyObject>)
+```
+
+Interpolated variables obey Python scope, as expected:
+
+```python
+In [9]: x = "global"
+   ...: def f():
+   ...:     x = "local"
+   ...:     ret = %julia py"x"
+   ...:     return ret
+   ...: f()
+Out[9]: 'local'
 ```
 
 #### IPython configuration

src/julia/magic.py

@@ -29,6 +29,12 @@
 from .core import Julia, JuliaError
 from .tools import redirect_output_streams
 
+try:
+    from IPython.core.magic import no_var_expand
+except ImportError:
+    def no_var_expand(f):
+        return f
+
 #-----------------------------------------------------------------------------
 # Main classes
 #-----------------------------------------------------------------------------
@@ -89,6 +95,7 @@ def __init__(self, shell):
         self._julia = Julia(init_julia=True)
         print()
 
+    @no_var_expand
     @line_cell_magic
     def julia(self, line, cell=None):
         """
@@ -97,14 +104,17 @@ def julia(self, line, cell=None):
         """
         src = compat.unicode_type(line if cell is None else cell)
 
-        try:
-            ans = self._julia.eval(src)
-        except JuliaError as e:
-            print(e, file=sys.stderr)
-            ans = None
-
-        return ans
-
+        # We assume the caller's frame is the first parent frame not in the
+        # IPython module. This seems to work with IPython back to ~v5, and
+        # is at least somewhat immune to future IPython internals changes,
+        # although by no means guaranteed to be perfect.
+        caller_frame = sys._getframe(3)
+        while caller_frame.f_globals.get('__name__').startswith("IPython"):
+            caller_frame = caller_frame.f_back
+
+        return self._julia.eval("""
+        _PyJuliaHelper.@prepare_for_pyjulia_call begin %s end
+        """%src)(self.shell.user_ns, caller_frame.f_locals)
 
 # Add to the global docstring the class information.
 __doc__ = __doc__.format(

src/julia/pyjulia_helper.jl

@@ -1,5 +1,9 @@
 module _PyJuliaHelper
 
+using PyCall
+using PyCall: pyeval_, Py_eval_input, Py_file_input
+using PyCall.MacroTools: isexpr, walk
+
 if VERSION < v"0.7-"
 nameof(m::Module) = ccall(:jl_module_name, Ref{Symbol}, (Any,), m)
 
@@ -44,6 +48,53 @@ if VERSION >= v"0.7-"
     end
 end
 
+
+macro prepare_for_pyjulia_call(ex)
+
+    # f(x) should return a transformed expression x and whether to recurse 
+    # into the new expression
+    function stoppable_walk(f, x)
+        (fx, recurse) = f(x)
+        walk(fx, (recurse ? (x -> stoppable_walk(f,x)) : identity), identity)
+    end
+
+    locals = gensym("locals")
+    globals = gensym("globals")
+
+    function make_pyeval(expr, options...)
+        code = string(expr)
+        T = length(options) == 1 && 'o' in options[1] ? PyObject : PyAny
+        input_type = '\n' in code ? Py_file_input : Py_eval_input
+        :($convert($T, $pyeval_($code, $(Expr(:$,globals)), $(Expr(:$,locals)), $input_type)))
+    end
+
+    ex = stoppable_walk(ex) do x
+        if isexpr(x, :$)
+            if isexpr(x.args[1], :$)
+                x.args[1], false
+            elseif x.args[1] isa Symbol
+                make_pyeval(x.args[1]), false
+            else
+                error("""syntax error in: \$($(string(x.args[1])))
+                Use py"..." instead of \$(...) for interpolating Python expressions, 
+                or \$\$(...) for a literal Julia \$(...).
+                """)
+            end
+        elseif isexpr(x, :macrocall) && x.args[1]==Symbol("@py_str")
+            # in Julia 0.7+, x.args[2] is a LineNumberNode, so filter it out
+            # in a way that's compatible with Julia 0.6:
+            make_pyeval(filter(s->(s isa String), x.args[2:end])...), false
+        else
+            x, true
+        end 
+    end
+
+    esc(quote
+        $pyfunction(($globals,$locals) -> (@eval Main $ex), $PyObject, $PyObject)
+    end)
+end
+
+
 module IOPiper
 
 const orig_stdin  = Ref{IO}()

test/test_magic.py

@@ -1,11 +1,36 @@
-from IPython.testing.globalipapp import get_ipython
-from julia import magic
+from textwrap import dedent
+
 import pytest
+from IPython import get_ipython as _get_ipython
+from IPython.testing.globalipapp import start_ipython as _start_ipython
+
+from julia import magic
 
 
+def get_ipython():
+    return _start_ipython() or _get_ipython()
+
 @pytest.fixture
 def julia_magics(julia):
-    return magic.JuliaMagics(shell=get_ipython())
+    julia_magics = magic.JuliaMagics(shell=get_ipython())
+
+    # a more conenient way to run strings (possibly with magic) as if they were
+    # an IPython cell
+    def run_cell(self, cell):
+        cell = dedent(cell).strip()
+        if cell.startswith("%%"):
+            return self.shell.run_cell_magic("julia","",cell.replace("%%julia","").strip())
+        else:
+            exec_result = self.shell.run_cell(cell)
+            if exec_result.error_in_exec:
+                raise exec_result.error_in_exec
+            else:
+                return exec_result.result
+
+    julia_magics.run_cell = run_cell.__get__(julia_magics, julia_magics.__class__)
+
+    return julia_magics
+
 
 
 def test_register_magics(julia):
@@ -23,14 +48,68 @@ def test_success_cell(julia_magics):
 
 
 def test_failure_line(julia_magics):
-    ans = julia_magics.julia('pop!([])')
-    assert ans is None
+    with pytest.raises(Exception):
+        julia_magics.julia('pop!([])')
 
 
 def test_failure_cell(julia_magics):
-    ans = julia_magics.julia(None, '1 += 1')
-    assert ans is None
-
+    with pytest.raises(Exception):
+        julia_magics.julia(None, '1 += 1')
+
+
+# Prior to IPython 7.3, $x did a string interpolation handled by IPython itself
+# for *line* magic, and could not be turned off. However, even prior to
+# IPython 7.3, *cell* magic never did the string interpolation, so below, any
+# time we need to test $x interpolation, do it as cell magic so it works on
+# IPython < 7.3
+
+def test_interp_var(julia_magics):
+    julia_magics.run_cell("x=1")
+    assert julia_magics.run_cell("""
+    %%julia
+    $x
+    """) == 1
+
+def test_interp_expr(julia_magics):
+    assert julia_magics.run_cell("""
+    x=1
+    %julia py"x+1"
+    """) == 2
+
+def test_bad_interp(julia_magics):
+    with pytest.raises(Exception):
+        assert julia_magics.run_cell("""
+        %julia $(x+1)
+        """)
+
+def test_string_interp(julia_magics):
+    assert julia_magics.run_cell("""
+    %%julia 
+    foo=3
+    "$foo"
+    """) == '3'
+
+def test_interp_escape(julia_magics):
+    assert julia_magics.run_cell("""
+    %%julia
+    bar=3
+    :($$bar)
+    """) == 3
+
+def test_type_conversion(julia_magics):
+    assert julia_magics.run_cell("""
+    %julia py"1" isa Integer && py"1"o isa PyObject
+    """) == True
+
+def test_scoping(julia_magics):
+    assert julia_magics.run_cell("""
+    x = "global"
+    def f():
+        x = "local"
+        ret = %julia py"x"
+        return ret
+    f()    
+    """) == "local"
 
 def test_revise_error():
     from julia.ipy import revise