-
Notifications
You must be signed in to change notification settings - Fork 105
/
default.nix
198 lines (189 loc) · 7.02 KB
/
default.nix
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
{
pkgs ? import <nixpkgs> { config = { allowUnfree = true; };},
lib ? pkgs.lib,
pythonInterpreters ? pkgs.useInterpreters or (with pkgs; [
python27
python35
python36
python37
python38
]),
...
}:
with builtins;
with lib;
let
commit = "1434cc0ee2da462f0c719d3a4c0ab4c87d0931e7";
fetchPypiSrc = builtins.fetchTarball {
name = "nix-pypi-fetcher-2";
url = "https://github.com/DavHau/nix-pypi-fetcher-2/archive/${commit}.tar.gz";
# Hash obtained using `nix-prefetch-url --unpack <url>`
sha256 = "080l189zzwrv75jgr7agvs4hjv4i613j86d4qky154fw5ncp0mnp";
};
fetchPypi = import (fetchPypiSrc);
patchDistutils = python_env:
with builtins;
let
verSplit = split "[\.]" python_env.python.version;
major = elemAt verSplit 0;
minor = elemAt verSplit 2;
lib_dir = "$out/lib/python${major}.${minor}";
site_pkgs_dir = "${lib_dir}/site-packages";
in
pkgs.symlinkJoin {
name = "${python_env.name}-patched";
paths = [ python_env ];
buildInputs = [
# prefer binary wrapper - but if that's not available (e.g. nixos 21.05)
# use the regular shell script wrapper
pkgs.makeBinaryWrapper or pkgs.makeWrapper
];
postBuild = ''
### Distutils
# symlinks to files
mkdir ${lib_dir}/distutils_tmp
cp -a ${lib_dir}/distutils/* ${lib_dir}/distutils_tmp/
rm ${lib_dir}/distutils
mv ${lib_dir}/distutils_tmp ${lib_dir}/distutils
# patch distutils/core.py
patch ${lib_dir}/distutils/core.py ${./distutils.patch}
# remove .pyc files
if [ ${major} = 2 ]; then
rm ${lib_dir}/distutils/core.pyc
else
chmod +w ${lib_dir}/distutils/__pycache__/
rm ${lib_dir}/distutils/__pycache__/core.*
fi
### Setuptools
# symlinks to files
mkdir ${site_pkgs_dir}/setuptools_tmp
cp -a ${site_pkgs_dir}/setuptools/* ${site_pkgs_dir}/setuptools_tmp/
rm ${site_pkgs_dir}/setuptools
mv ${site_pkgs_dir}/setuptools_tmp ${site_pkgs_dir}/setuptools
# patch setuptools/__init__.py
echo ${site_pkgs_dir}/setuptools/__init__.py
patch ${site_pkgs_dir}/setuptools/__init__.py ${./setuptools.patch}
# remove .pyc files
if [ ${major} = 2 ]; then
rm ${site_pkgs_dir}/setuptools/__init__.pyc
else
chmod +w ${site_pkgs_dir}/setuptools/__pycache__
rm ${site_pkgs_dir}/setuptools/__pycache__/__init__.*
fi
# fix executables
shopt -s nullglob
for f in ${python_env}/bin/*; do
f=$(basename "$f")
# wrap it once more, set PYTHONPATH, ignoring NIXPYTHON_PATH and NIX_PYTHONEXECUTABLE
rm "$out/bin/$f" # remove the existing symlink
makeWrapper "${python_env}/bin/$f" "$out/bin/$f" \
--set PYTHONPATH "$out/lib/python${major}.${minor}"
done
'';
};
mkPy = python:
let
python_env = python.withPackages (ps: with ps; [
# base requirements
setuptools
]);
in
patchDistutils python_env;
# This is how pip invokes setup.py. We do this manually instead of using pip to increase performance by ~40%
setuptools_shim = ''
import sys, setuptools, tokenize, os; sys.argv[0] = 'setup.py'; __file__='setup.py';
f=getattr(tokenize, 'open', open)(__file__);
code=f.read().replace('\r\n', '\n');
f.close();
exec(compile(code, __file__, 'exec'))
'';
# note on SETUPTOOLS_USE_DISTUTILS=stdlib: Restore old setuptools behaviour (since
# https://github.com/pypa/setuptools/commit/b6fcbbd00cb6d5607c9272dec452a50457bdb292),
# to keep it working with mach-nix.
script = pyVersions: ''
mkdir $out
${concatStringsSep "\n" (forEach pythonInterpreters (interpreter:
let
py = mkPy interpreter;
verSplit = splitString "." interpreter.version;
major = elemAt verSplit 0;
minor = elemAt verSplit 1;
v = "${major}${minor}";
# only use selected interpreters
in optionalString (pyVersions == [] || elem v pyVersions) ''
echo "extracting metadata for python${v}"
SETUPTOOLS_USE_DISTUTILS=stdlib out_file=$out/python${v}.json ${py}/bin/python -c "${setuptools_shim}" install &> $out/python${v}.log || true
''
))}
'';
script_single = py: ''
chmod +x setup.py || true
mkdir $out
echo "extracting dependencies"
SETUPTOOLS_USE_DISTUTILS=stdlib out_file=$out/python.json ${py}/bin/python -c "${setuptools_shim}" install &> $out/python.log || true
cat $out/python.log
'';
base_derivation = pyVersions: with pkgs; {
buildInputs = [ unzip pkg-config ];
phases = ["unpackPhase" "installPhase"];
# Tells our modified python builtins to dump setup attributes instead of doing an actual installation
dump_setup_attrs = "y";
PYTHONIOENCODING = "utf8"; # My gut feeling is that encoding issues might decrease by this
LANG = "C.utf8";
installPhase = script pyVersions;
};
sanitizeDerivationName = string: lib.pipe string [
# Get rid of string context. This is safe under the assumption that the
# resulting string is only used as a derivation name
unsafeDiscardStringContext
# Strip all leading "."
(x: elemAt (match "\\.*(.*)" x) 0)
# Split out all invalid characters
# https://github.com/NixOS/nix/blob/2.3.2/src/libstore/store-api.cc#L85-L112
# https://github.com/NixOS/nix/blob/2242be83c61788b9c0736a92bb0b5c7bbfc40803/nix-rust/src/store/path.rs#L100-L125
(split "[^[:alnum:]+._?=-]+")
# Replace invalid character ranges with a "-"
(concatMapStrings (s: if lib.isList s then "-" else s))
# Limit to 211 characters (minus 4 chars for ".drv")
(x: substring (lib.max (stringLength x - 207) 0) (-1) x)
# If the result is empty, replace it with "unknown"
(x: if stringLength x == 0 then "unknown" else x)
];
in
with pkgs;
rec {
inherit machnix_source mkPy pythonInterpreters;
example = extractor {pkg = "requests"; version = "2.22.0";};
extract_from_src = {py, src, name}:
let
py' = if isString py then pkgs."${py}" else py;
in
stdenv.mkDerivation ( (base_derivation []) // {
inherit src;
name = "${name}-mach-nix-requirements";
installPhase = script_single (mkPy py');
});
extractor = {pkg, version, ...}:
stdenv.mkDerivation ({
name = "${pkg}-${version}-requirements";
src = fetchPypi pkg version;
} // (base_derivation []));
extractor-fast = {pkg, version, url, sha256, pyVersions ? [], ...}:
stdenv.mkDerivation ( rec {
name = sanitizeDerivationName "${pkg}-${version}-requirements";
src = (pkgs.fetchurl {
inherit url sha256;
}).overrideAttrs (_: {
name = sanitizeDerivationName _.name;
});
} // (base_derivation pyVersions));
make-drvs =
let
jobs = fromJSON (readFile (getEnv "EXTRACTOR_JOBS_JSON_FILE"));
results = listToAttrs (map (job:
nameValuePair
"${job.pkg}#${job.version}"
(extractor-fast job).drvPath
) jobs);
in toJSON results;
}