From bf16d03075bbfbb7078f396cec6a782b57cc12ae Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Mon, 19 Jan 2015 20:02:47 -0500 Subject: [PATCH] ruby: support for proper bundler installs --- .../interpreters/ruby/bundler-env.nix | 72 ++++++--- pkgs/development/interpreters/ruby/gem.nix | 2 +- .../interpreters/ruby/monkey_patches.rb | 146 ++++++++++++++++++ pkgs/top-level/all-packages.nix | 4 +- 4 files changed, 203 insertions(+), 21 deletions(-) create mode 100644 pkgs/development/interpreters/ruby/monkey_patches.rb diff --git a/pkgs/development/interpreters/ruby/bundler-env.nix b/pkgs/development/interpreters/ruby/bundler-env.nix index 3fb02fa1ef8..bf2b4733c39 100644 --- a/pkgs/development/interpreters/ruby/bundler-env.nix +++ b/pkgs/development/interpreters/ruby/bundler-env.nix @@ -1,21 +1,34 @@ -{ stdenv, runCommand, writeText, writeScriptBin, ruby, lib, callPackage -, gemFixes, fetchurl, fetchgit, buildRubyGem +{ stdenv, runCommand, writeText, writeScript, writeScriptBin, ruby, lib +, callPackage , gemFixes, fetchurl, fetchgit, buildRubyGem +#, bundler_PATCHED +, bundler_HEAD +, git }@defs: # This is a work-in-progress. -# The idea is that his will replace load-ruby-env.nix, -# using (a patched) bundler to handle the entirety of the installation process. +# The idea is that his will replace load-ruby-env.nix. { name, gemset, gemfile, lockfile, ruby ? defs.ruby, fixes ? gemFixes }@args: let const = x: y: x; + #bundler = bundler_PATCHED; + bundler = bundler_HEAD.override { inherit ruby; }; + inherit (builtins) attrValues; fetchers.path = attrs: attrs.src.path; - fetchers.gem = attrs: fetchurl { - url = "${attrs.src.source or "https://rubygems.org"}/downloads/${attrs.name}-${attrs.version}.gem"; - inherit (attrs.src) sha256; - }; + fetchers.gem = attrs: + let fname = "${attrs.name}-${attrs.version}.gem"; + in toString (runCommand fname { + gem = fetchurl { + url = "${attrs.src.source or "https://rubygems.org"}/downloads/${fname}"; + inherit (attrs.src) sha256; + }; + } '' + mkdir $out + cp $gem $out/${fname} + '') + "/${fname}"; + fetchers.git = attrs: fetchgit { inherit (attrs.src) url rev sha256 fetchSubmodules; leaveDotGit = true; @@ -40,6 +53,13 @@ let instantiate (attrs // { inherit name; }) ); + # only the *.gem files. + gems = lib.fold (next: acc: + if next.source.type == "gem" + then acc ++ [next.src] + else acc + ) [] (attrValues instantiated); + runRuby = name: env: command: runCommand name env '' ${ruby}/bin/ruby ${writeText name command} @@ -102,44 +122,58 @@ let ''; # rewrite PATH sources to point into the nix store. - pureLockfile = runRuby "pureLockfile" { inherit sources; } '' - out = ENV['out'] - paths = eval(File.read(ENV['sources'])) + purifyLockfile = writeScript "purifyLockfile" '' + #!${ruby}/bin/ruby - lockfile = File.read("${lockfile}") + out = ENV['out'] + sources = eval(File.read("${sources}")) + paths = sources["path"] + + lockfile = STDIN.read paths.each_pair do |impure, pure| lockfile.gsub!(/^ remote: #{Regexp.escape(impure)}/, " remote: #{pure}") end - File.open(out, "wb") do |f| - f.print lockfile - end + print lockfile ''; in stdenv.mkDerivation { inherit name; + buildInputs = [ + ruby + bundler + git + ]; + phases = [ "installPhase" "fixupPhase" ]; outputs = [ "out" # the installed libs/bins "bundler" # supporting files for bundler ]; - phases = [ "installPhase" "fixupPhase" ]; installPhase = '' # Copy the Gemfile and Gemfile.lock mkdir -p $bundler - BUNDLE_GEMFILE=$bundler/Gemfile + export BUNDLE_GEMFILE=$bundler/Gemfile cp ${gemfile} $BUNDLE_GEMFILE - cp ${pureLockfile} $BUNDLE_GEMFILE.lock + cat ${lockfile} | ${purifyLockfile} > $BUNDLE_GEMFILE.lock export NIX_GEM_SOURCES=${sources} + export NIX_BUNDLER_GEMPATH=${bundler}/${ruby.gemPath} export GEM_HOME=$out/${ruby.gemPath} export GEM_PATH=$GEM_HOME mkdir -p $GEM_HOME - bundler install + mkdir gems + for gem in ${toString gems}; do + ln -s $gem gems + done + + cp ${./monkey_patches.rb} monkey_patches.rb + export RUBYOPT="-rmonkey_patches.rb -I $(pwd -P)" + bundler install --frozen ''; passthru = { inherit ruby; diff --git a/pkgs/development/interpreters/ruby/gem.nix b/pkgs/development/interpreters/ruby/gem.nix index 5516646f920..dd6338cd813 100644 --- a/pkgs/development/interpreters/ruby/gem.nix +++ b/pkgs/development/interpreters/ruby/gem.nix @@ -107,9 +107,9 @@ stdenv.mkDerivation (attrs // { makeWrapper $prog $out/bin/$(basename $prog) \ --prefix GEM_PATH : "$out/${ruby.gemPath}:$GEM_PATH" \ --prefix RUBYLIB : "${rubygems}/lib" \ - --set RUBYOPT rubygems \ $extraWrapperFlags ''${extraWrapperFlagsArray[@]} done + #--prefix RUBYOPT rubygems \ # looks like useless files which break build repeatability and consume space rm -fv $out/${ruby.gemPath}/doc/*/*/created.rid || true diff --git a/pkgs/development/interpreters/ruby/monkey_patches.rb b/pkgs/development/interpreters/ruby/monkey_patches.rb new file mode 100644 index 00000000000..694a199a610 --- /dev/null +++ b/pkgs/development/interpreters/ruby/monkey_patches.rb @@ -0,0 +1,146 @@ +require 'bundler' + +Bundler.module_eval do + class << self + # mappings from original uris to store paths. + def nix_gem_sources + @nix_gem_sources ||= + begin + src = ENV['NIX_GEM_SOURCES'] + eval(Bundler.read_file(src)) + end + end + + # extract the gemspecs from the gems pulled from Rubygems. + def nix_gemspecs + @nix_gemspecs ||= Dir.glob("gems/*.gem").map do |path| + Bundler.rubygems.spec_from_gem(path) + end + end + + # map a git uri to a fetchgit store path. + def nix_git(uri) + Pathname.new(nix_gem_sources["git"][uri]) + end + end +end + +Bundler::Source::Git::GitProxy.class_eval do + def checkout + unless path.exist? + FileUtils.mkdir_p(path.dirname) + FileUtils.cp_r(Bundler.nix_git(@uri).join(".git"), path) + system("chmod -R +w #{path}") + end + end + + def copy_to(destination, submodules=false) + unless File.exist?(destination.join(".git")) + FileUtils.mkdir_p(destination.dirname) + FileUtils.cp_r(Bundler.nix_git(@uri), destination) + system("chmod -R +w #{destination}") + end + end +end + +Bundler::Fetcher.class_eval do + def use_api + true + end + + def fetch_dependency_remote_specs(gem_names) + Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(',')}" + deps_list = [] + + spec_list = gem_names.map do |name| + spec = Bundler.nix_gemspecs.detect {|spec| spec.name == name } + dependencies = spec.dependencies. + select {|dep| dep.type != :development}. + map do |dep| + deps_list << dep.name + dep + end + + [spec.name, spec.version, spec.platform, dependencies] + end + + [spec_list, deps_list.uniq] + end +end + +Bundler::Source::Rubygems.class_eval do + # We copy all gems into $PWD/gems, and this allows RubyGems to find those + # gems during installation. + def fetchers + @fetchers ||= [ + Bundler::Fetcher.new(URI.parse("file://#{File.expand_path(Dir.pwd)}")) + ] + end + + # Look-up gems that were originally from Rubygems. + def remote_specs + @remote_specs ||= + begin + lockfile = Bundler::LockfileParser.new(Bundler.read_file(Bundler.default_lockfile)) + gem_names = lockfile.specs. + select {|spec| spec.source.is_a?(Bundler::Source::Rubygems)}. + map {|spec| spec.name} + idx = Bundler::Index.new + api_fetchers.each do |f| + Bundler.ui.info "Fetching source index from #{f.uri}" + idx.use f.specs(gem_names, self) + end + idx + end + end +end + +Gem::Installer.class_eval do + # Make the wrappers automagically use bundler. + # + # Stage 1. + # Set $BUNDLE_GEMFILE so bundler knows what gems to load. + # Set $GEM_HOME to the installed gems, because bundler looks there for + # non-Rubygems installed gems (e.g. git/svn/path sources). + # Set $GEM_PATH to include both bundler and installed gems. + # + # Stage 2. + # Setup bundler, locking down the gem versions. + # + # Stage 3. + # Reset $BUNDLE_GEMFILE, $GEM_HOME, $GEM_PATH. + # + # Stage 4. + # Run the actual executable. + def app_script_text(bin_file_name) + return <<-TEXT +#{shebang bin_file_name} +# +# This file was generated by Nix's RubyGems. +# +# The application '#{spec.name}' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +old_gemfile = ENV["BUNDLE_GEMFILE"] +old_gem_home = ENV["GEM_HOME"] +old_gem_path = ENV["GEM_PATH"] + +ENV["BUNDLE_GEMFILE"] = + "#{ENV["BUNDLE_GEMFILE"]}" +ENV["GEM_HOME"] = + "#{ENV["GEM_HOME"]}" +ENV["GEM_PATH"] = + "#{ENV["NIX_BUNDLER_GEMPATH"]}:#{ENV["GEM_HOME"]}\#{old_gem_path ? ":\#{old_gem_path}" : ""}}" + +require 'rubygems' +require 'bundler/setup' + +ENV["BUNDLE_GEMFILE"] = old_gemfile +ENV["GEM_HOME"] = old_gem_home +ENV["GEM_PATH"] = old_gem_path + +load Gem.bin_path('#{spec.name}', '#{bin_file_name}') +TEXT + end +end diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index a267110a771..57d65179004 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -4189,7 +4189,9 @@ let }; bundler = callPackage ../development/interpreters/ruby/bundler.nix { }; - bundler_HEAD = callPackage ../development/interpreters/ruby/bundler-head.nix { }; + bundler_HEAD = import ../development/interpreters/ruby/bundler-head.nix { + inherit buildRubyGem coreutils fetchgit; + }; gemFixes = callPackage ../development/interpreters/ruby/fixes.nix { }; buildRubyGem = callPackage ../development/interpreters/ruby/gem.nix { }; loadRubyEnv = callPackage ../development/interpreters/ruby/load-ruby-env.nix { };