Skip to content

Commit

Permalink
Merge pull request #216 from mkitti/mk/conda_jl_conda_exe
Browse files Browse the repository at this point in the history 
Introduce CONDA_JL_CONDA_EXE environment variable
  • Loading branch information
@isuruf
isuruf authored Feb 8, 2022
2 parents f169018 + 84c662c commit 8f71332
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 46 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name = "Conda"
uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d"
version = "1.6"
version = "1.7"

[deps]
Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
Expand Down
17 changes: 15 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
[![Build Status](https://github.com/JuliaPy/Conda.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaPy/Conda.jl/actions/workflows/CI.yml)

This package allows one to use [conda](http://conda.pydata.org/) as a cross-platform binary provider for Julia for other Julia packages,
especially to install binaries that have complicated dependencies
like Python.
especially to install binaries that have complicated dependencies like Python.

`conda` is a package manager which started as the binary package manager for the
Anaconda Python distribution, but it also provides arbitrary packages. Instead
Expand Down Expand Up @@ -81,6 +80,20 @@ julia> ENV["CONDA_JL_HOME"] = "/path/to/miniconda/envs/conda_jl" # change this
pkg> build Conda
```

## Using a conda executable outside of the home environment
To use a specific conda executable, set the `CONDA_JL_CONDA_EXE` environment
variable to the location of the conda executable. This conda executable can
exist outside of the environment set by `CONDA_JL_HOME`. To apply the settting,
rebuild `Conda.jl`. In Julia, run:

```jl
julia> ENV["CONDA_JL_CONDA_EXE"] = "/path/to/miniconda/bin/conda" # change this to the path of the conda executable

pkg> build Conda
```

*The use of `CONDA_JL_CONDA_EXE` requires at least version 1.7 of Conda.jl.*

## Conda and pip
As of [conda 4.6.0](https://docs.conda.io/projects/conda/en/latest/user-guide/configuration/pip-interoperability.html#improving-interoperability-with-pip) there is improved support for PyPi packages.
**Conda is still the recommended installation method** however if there are packages that are only availible with `pip` one can do the following:
Expand Down
42 changes: 42 additions & 0 deletions deps/build.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,19 @@ Using the Anaconda/defaults channel instead, which is free for non-commercial us
if !isdefined(@__MODULE__, :USE_MINIFORGE)
const USE_MINIFORGE = USE_MINIFORGE_DEFAULT
end
function default_conda_exe(ROOTENV)
@static if Sys.iswindows()
p = joinpath(ROOTENV, "Scripts")
conda_bat = joinpath(p, "conda.bat")
isfile(conda_bat) ? conda_bat : joinpath(p, "conda.exe")
else
joinpath(ROOTENV, "bin", "conda")
end
end

if !isdefined(@__MODULE__, :CONDA_EXE)
const CONDA_EXE = default_conda_exe(ROOTENV)
end
end

MINICONDA_VERSION = get(ENV, "CONDA_JL_VERSION", DefaultDeps.MINICONDA_VERSION)
Expand Down Expand Up @@ -52,10 +65,39 @@ These will require rebuilding.
""")
end

CONDA_EXE = get(ENV, "CONDA_JL_CONDA_EXE") do
if ROOTENV == DefaultDeps.ROOTENV
DefaultDeps.CONDA_EXE
else
DefaultDeps.default_conda_exe(ROOTENV)
end
end

if haskey(ENV, "CONDA_JL_CONDA_EXE")
# Check to see if CONDA_EXE is an executable file
if isfile(CONDA_EXE)
if Sys.isexecutable(CONDA_EXE)
@info "Executable conda located." CONDA_EXE
else
error("CONDA_JL_CONDA_EXE, $CONDA_EXE, cannot be executed by the current user.")
end
else
error("CONDA_JL_CONDA_EXE, $CONDA_EXE, does not exist.")
end
else
if !isfile(CONDA_EXE)
# An old CONDA_EXE has gone missing, revert to default in ROOTENV
@info "CONDA_EXE not found. Reverting to default in ROOTENV" CONDA_EXE ROOTENV
CONDA_EXE = DefaultDeps.default_conda_exe(ROOTENV)
end
end


deps = """
const ROOTENV = "$(escape_string(ROOTENV))"
const MINICONDA_VERSION = "$(escape_string(MINICONDA_VERSION))"
const USE_MINIFORGE = $USE_MINIFORGE
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""

mkpath(condadir)
Expand Down
19 changes: 11 additions & 8 deletions src/Conda.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,12 @@ function python_dir(env::Environment)
end
const PYTHONDIR = python_dir(ROOTENV)

# note: the same conda program is used for all environments
const conda = if Sys.iswindows()
p = script_dir(ROOTENV)
conda_bat = joinpath(p, "conda.bat")
isfile(conda_bat) ? conda_bat : joinpath(p, "conda.exe")
else
joinpath(bin_dir(ROOTENV), "conda")
if ! @isdefined(CONDA_EXE)
# We have an oudated deps.jl file that does not define CONDA_EXE
error("CONDA_EXE not defined in $deps_file.\nPlease rebuild Conda.jl via `using Pkg; pkg\"build Conda\";`")
end
# note: the same conda program is used for all environments
const conda = CONDA_EXE

"Path to the condarc file"
function conda_rc(env::Environment)
Expand Down Expand Up @@ -191,6 +189,7 @@ _quiet() = get(ENV, "CI", "false") == "true" ? `-q` : ``
"Install miniconda if it hasn't been installed yet; _install_conda(true) installs Conda even if it has already been installed."
function _install_conda(env::Environment, force::Bool=false)
if force || !isfile(Conda.conda)
@assert startswith(abspath(Conda.conda), abspath(PREFIX)) "CONDA_EXE, $(conda), does not exist within $PREFIX"
@info("Downloading miniconda installer ...")
if Sys.isunix()
installer = joinpath(PREFIX, "installer.sh")
Expand All @@ -211,7 +210,7 @@ function _install_conda(env::Environment, force::Bool=false)
end
end
if !isdir(prefix(env))
runconda(`create $(_quiet()) -y -p $(prefix(env))`)
create(env)
end
end

Expand All @@ -228,6 +227,10 @@ function rm(pkg::PkgOrPkgs, env::Environment=ROOTENV)
runconda(`remove $(_quiet()) -y $pkg`, env)
end

function create(env::Environment)
runconda(`create $(_quiet()) -y -p $(prefix(env))`)
end

"Update all installed packages."
function update(env::Environment=ROOTENV)
if env == ROOTENV
Expand Down
107 changes: 72 additions & 35 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,36 +70,42 @@ Conda.rm_channel("foo", env)
Conda.add("zlib", env; channel=alt_channel)

@testset "Batch install and uninstall" begin
Conda.add(["affine", "ansi2html"], env)
installed = Conda._installed_packages(env)
@test "affine" ∈ installed
@test "ansi2html" ∈ installed

Conda.rm(["affine", "ansi2html"], env)
installed = Conda._installed_packages(env)
@test "affine" ∉ installed
@test "ansi2html" ∉ installed
mktempdir() do env
Conda.create(env)
Conda.add(["affine", "ansi2html"], env)
installed = Conda._installed_packages(env)
@test "affine" ∈ installed
@test "ansi2html" ∈ installed

Conda.rm(["affine", "ansi2html"], env)
installed = Conda._installed_packages(env)
@test "affine" ∉ installed
@test "ansi2html" ∉ installed
end
end

# Run conda clean
Conda.clean(; debug=true)

@testset "Exporting and creating environments" begin
new_env = :test_conda_jl_2
Conda.add("curl", env)
Conda.export_list("conda-pkg.txt", env)

# Create a new environment
rm(Conda.prefix(new_env); force=true, recursive=true)
Conda.import_list(
IOBuffer(read("conda-pkg.txt")), new_env; channels=["foo", alt_channel, default_channel]
)

# Ensure that our new environment has our channels and package installed.
@test Conda.channels(new_env) == ["foo", alt_channel, default_channel]
installed = Conda._installed_packages(new_env)
@test "curl" ∈ installed
rm("conda-pkg.txt")
mktempdir() do env
new_env = :test_conda_jl_2
Conda.create(env)
Conda.add("curl", env)
Conda.export_list("conda-pkg.txt", env)

# Create a new environment
rm(Conda.prefix(new_env); force=true, recursive=true)
Conda.import_list(
IOBuffer(read("conda-pkg.txt")), new_env; channels=["foo", alt_channel, default_channel]
)

# Ensure that our new environment has our channels and package installed.
@test Conda.channels(new_env) == ["foo", alt_channel, default_channel]
installed = Conda._installed_packages(new_env)
@test "curl" ∈ installed
rm("conda-pkg.txt")
end
end

@testset "Conda.pip_interop" begin
Expand Down Expand Up @@ -150,6 +156,17 @@ end
end
end

function default_conda_exe(ROOTENV)
@static if Sys.iswindows()
p = joinpath(ROOTENV, "Scripts")
conda_bat = joinpath(p, "conda.bat")
isfile(conda_bat) ? conda_bat : joinpath(p, "conda.exe")
else
joinpath(ROOTENV, "bin", "conda")
end
end


if Sys.ARCH in [:x86, :i686]
CONDA_JL_USE_MINIFORGE_DEFAULT = "false"
else
Expand All @@ -162,12 +179,15 @@ end
@test !isfile(depsfile)
@test !isfile(joinpath(condadir, "deps.jl"))

withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => nothing) do
withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => nothing, "CONDA_JL_CONDA_EXE" => nothing) do
Pkg.build("Conda")
local ROOTENV=joinpath(condadir, "3")
local CONDA_EXE=default_conda_exe(ROOTENV)
@test read(depsfile, String) == """
const ROOTENV = "$(escape_string(joinpath(condadir, "3")))"
const ROOTENV = "$(escape_string(ROOTENV))"
const MINICONDA_VERSION = "3"
const USE_MINIFORGE = $(CONDA_JL_USE_MINIFORGE_DEFAULT)
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""
end
end
Expand All @@ -179,12 +199,15 @@ end
@test !isfile(depsfile)
@test !isfile(joinpath(condadir, "deps.jl"))

withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => "1") do
withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => "1", "CONDA_JL_CONDA_EXE" => nothing) do
Pkg.build("Conda")
local ROOTENV=joinpath(condadir, "3")
local CONDA_EXE=default_conda_exe(ROOTENV)
@test read(depsfile, String) == """
const ROOTENV = "$(escape_string(joinpath(condadir, "3")))"
const MINICONDA_VERSION = "3"
const USE_MINIFORGE = true
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""
end
end
Expand All @@ -193,12 +216,15 @@ end
@test !isfile(depsfile)
@test !isfile(joinpath(condadir, "deps.jl"))

withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => "0") do
withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => "0", "CONDA_JL_CONDA_EXE" => nothing) do
Pkg.build("Conda")
local ROOTENV=joinpath(condadir, "3")
local CONDA_EXE=default_conda_exe(ROOTENV)
@test read(depsfile, String) == """
const ROOTENV = "$(escape_string(joinpath(condadir, "3")))"
const ROOTENV = "$(escape_string(ROOTENV))"
const MINICONDA_VERSION = "3"
const USE_MINIFORGE = false
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""
end
end
Expand All @@ -207,12 +233,14 @@ end
@testset "custom home" begin
preserve_build() do
mktempdir() do dir
withenv("CONDA_JL_VERSION" => "3", "CONDA_JL_HOME" => dir, "CONDA_JL_USE_MINIFORGE" => nothing) do
withenv("CONDA_JL_VERSION" => "3", "CONDA_JL_HOME" => dir, "CONDA_JL_USE_MINIFORGE" => nothing, "CONDA_JL_CONDA_EXE" => nothing) do
Pkg.build("Conda")
local CONDA_EXE=default_conda_exe(dir)
@test read(depsfile, String) == """
const ROOTENV = "$(escape_string(dir))"
const MINICONDA_VERSION = "3"
const USE_MINIFORGE = $(CONDA_JL_USE_MINIFORGE_DEFAULT)
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""
end
end
Expand All @@ -222,28 +250,37 @@ end
@testset "version mismatch" begin
preserve_build() do
# Mismatch in written file
local ROOTENV=joinpath(condadir, "3")
local CONDA_EXE=default_conda_exe(ROOTENV)
write(depsfile, """
const ROOTENV = "$(escape_string(joinpath(condadir, "3")))"
const ROOTENV = "$(escape_string(ROOTENV))"
const MINICONDA_VERSION = "2"
const USE_MINIFORGE = $(CONDA_JL_USE_MINIFORGE_DEFAULT)
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
""")

withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => nothing) do
withenv("CONDA_JL_VERSION" => nothing, "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => nothing, "CONDA_JL_CONDA_EXE" => nothing) do
Pkg.build("Conda")
local ROOTENV=joinpath(condadir, "2")
local CONDA_EXE=default_conda_exe(ROOTENV)
@test read(depsfile, String) == """
const ROOTENV = "$(escape_string(joinpath(condadir, "2")))"
const ROOTENV = "$(escape_string(ROOTENV))"
const MINICONDA_VERSION = "2"
const USE_MINIFORGE = $(CONDA_JL_USE_MINIFORGE_DEFAULT)
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""
end

# ROOTENV should be replaced since CONDA_JL_HOME wasn't explicitly set
withenv("CONDA_JL_VERSION" => "3", "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => nothing) do
withenv("CONDA_JL_VERSION" => "3", "CONDA_JL_HOME" => nothing, "CONDA_JL_USE_MINIFORGE" => nothing, "CONDA_JL_CONDA_EXE" => nothing) do
Pkg.build("Conda")
local ROOTENV=joinpath(condadir, "3")
local CONDA_EXE=default_conda_exe(ROOTENV)
@test read(depsfile, String) == """
const ROOTENV = "$(escape_string(joinpath(condadir, "3")))"
const ROOTENV = "$(escape_string(ROOTENV))"
const MINICONDA_VERSION = "3"
const USE_MINIFORGE = $(CONDA_JL_USE_MINIFORGE_DEFAULT)
const CONDA_EXE = "$(escape_string(CONDA_EXE))"
"""
end
end
Expand Down

4 comments on commit 8f71332

@mkitti
Copy link
Member

@mkitti mkitti commented on 8f71332 Feb 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@isuruf any objection to registering a new release? I think we need this to fix the issue in conda-forge/pyjulia-feedstock#1

@isuruf
Copy link
Collaborator Author

@isuruf isuruf commented on 8f71332 Feb 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/54566

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v1.7.0 -m "<description of version>" 8f7133206f3efb6308dff5a2b09393d10e6cc122
git push origin v1.7.0

@mkitti
Copy link
Member

@mkitti mkitti commented on 8f71332 Feb 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you!

Please sign in to comment.