[release-0.5] backports for 0.5.0-rc3

.travis.yml

@@ -81,7 +81,7 @@ before_install:
         brew install -v staticfloat/juliadeps/libgfortran;
         brew install -v --only-dependencies --HEAD julia;
         BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm37-julia)/bin/llvm-config-3.7.1 LLVM_SIZE=$(brew --prefix llvm37-julia)/bin/llvm-size-3.7.1";
-        BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1 STAGE2_DEPS=utf8proc";
+        BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1";
         BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas";
         for lib in LLVM SUITESPARSE ARPACK BLAS FFTW LAPACK GMP MPFR PCRE LIBUNWIND; do
             export BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1";
@@ -111,6 +111,10 @@ script:
     - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 && cd /tmp/julia/share/julia/test &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online pkg
-    - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia && rm -rf julia/deps/build/julia-env
+    - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia &&
+        rm -rf julia/deps/build/julia-env &&
+        rm -rf julia/deps/build/libssh2-*/CMakeFiles/Makefile2 &&
+        rm -rf julia/deps/build/curl-*/config.log &&
+        rm -rf julia/deps/build/curl-*/libtool
 # uncomment the following if failures are suspected to be due to the out-of-memory killer
 #    - dmesg

LICENSE.md

@@ -57,6 +57,7 @@ their own licenses:
 - [FFTW](http://fftw.org/doc/License-and-Copyright.html) [GPL2+]
 - [GMP](http://gmplib.org/manual/Copying.html#Copying) [LGPL3+ or GPL2+]
 - [LIBGIT2](https://github.com/libgit2/libgit2/blob/development/COPYING) [GPL2+ with unlimited linking exception]
+- [CURL](https://curl.haxx.se/docs/copyright.html) [MIT/X derivative]
 - [LIBSSH2](https://github.com/libssh2/libssh2/blob/master/COPYING) [BSD-3]
 - [MBEDTLS](https://tls.mbed.org/how-to-get) [either GPLv2 or Apache 2.0]
 - [MPFR](http://www.mpfr.org/mpfr-current/mpfr.html#Copying) [LGPL3+]

Make.inc

@@ -38,6 +38,7 @@ USE_SYSTEM_LIBUV:=0
 USE_SYSTEM_UTF8PROC:=0
 USE_SYSTEM_MBEDTLS:=0
 USE_SYSTEM_LIBSSH2:=0
+USE_SYSTEM_CURL:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
 
@@ -339,6 +340,11 @@ endif
 
 STDLIBCPP_FLAG :=
 
+ifeq ($(OS), FreeBSD)
+USEGCC := 0
+USECLANG := 1
+endif
+
 ifeq ($(OS), Darwin)
 DARWINVER := $(shell uname -r | cut -b 1-2)
 DARWINVER_GTE13 := $(shell expr `uname -r | cut -b 1-2` \>= 13)
@@ -459,6 +465,7 @@ endif #ARCH
 LD := link
 endif #USEMSVC
 RANLIB := $(CROSS_COMPILE)ranlib
+OBJCOPY := $(CROSS_COMPILE)objcopy
 
 # file extensions
 ifeq ($(OS), WINNT)

Makefile

@@ -277,6 +277,9 @@ endif
 ifeq ($(USE_SYSTEM_LIBSSH2),0)
 JL_PRIVATE_LIBS += ssh2
 endif
+ifeq ($(USE_SYSTEM_CURL),0)
+JL_PRIVATE_LIBS += curl
+endif
 ifeq ($(USE_SYSTEM_LIBGIT2),0)
 JL_PRIVATE_LIBS += git2
 endif
@@ -468,7 +471,11 @@ endif
 
 ifeq ($(OS), WINNT)
 	[ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \
-		cp 7z.exe 7z.dll busybox.exe libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin )
+		cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll libgfortran-3.dll libquadmath-0.dll libstdc++-6.dll libgcc_s_s*-1.dll libssp-0.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin )
+ifeq ($(USE_GPL_LIBS), 1)
+	[ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \
+		cp busybox.exe $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin )
+endif
 	cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe
 
 	# create file listing for uninstall. note: must have Windows path separators and line endings.
@@ -593,7 +600,14 @@ ifneq (,$(filter $(ARCH), i386 i486 i586 i686))
 	$(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920.exe && \
 	7z x -y 7z920.exe 7z.exe 7z.dll && \
 	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_13.2 \
-		"mingw32-libgfortran3 mingw32-libquadmath0 mingw32-libstdc++6 mingw32-libgcc_s_sjlj1 mingw32-libssp0 mingw32-libexpat1 mingw32-zlib1" && \
+		"mingw32-libexpat1 mingw32-zlib1" && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libgfortran3-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libquadmath0-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libstdc++6-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libgcc_s_sjlj1-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw32-libssp0-5.3.0-1.1.noarch.rpm && \
+	for i in *.rpm; do 7z x -y $$i; done && \
+	for i in *.cpio; do 7z x -y $$i; done && \
 	cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll .
 else ifeq ($(ARCH),x86_64)
 	cd $(JULIAHOME)/dist-extras && \
@@ -602,20 +616,30 @@ else ifeq ($(ARCH),x86_64)
 	mv _7z.dll 7z.dll && \
 	mv _7z.exe 7z.exe && \
 	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_13.2 \
-		"mingw64-libgfortran3 mingw64-libquadmath0 mingw64-libstdc++6 mingw64-libgcc_s_seh1 mingw64-libssp0 mingw64-libexpat1 mingw64-zlib1" && \
+		"mingw64-libexpat1 mingw64-zlib1" && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libgfortran3-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libquadmath0-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libstdc++6-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libgcc_s_seh1-5.3.0-1.1.noarch.rpm && \
+	$(JLDOWNLOAD) https://julialang.s3.amazonaws.com/bin/winnt/extras/mingw64-libssp0-5.3.0-1.1.noarch.rpm && \
+	for i in *.rpm; do 7z x -y $$i; done && \
+	for i in *.cpio; do 7z x -y $$i; done && \
 	cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll .
 else
 	$(error no win-extras target for ARCH=$(ARCH))
 endif
 	cd $(JULIAHOME)/dist-extras && \
 	$(JLDOWNLOAD) http://downloads.sourceforge.net/sevenzip/7z920_extra.7z && \
 	$(JLDOWNLOAD) https://unsis.googlecode.com/files/nsis-2.46.5-Unicode-setup.exe && \
-	$(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \
 	chmod a+x 7z.exe && \
 	chmod a+x 7z.dll && \
 	$(call spawn,./7z.exe) x -y -onsis nsis-2.46.5-Unicode-setup.exe && \
-	chmod a+x ./nsis/makensis.exe && \
+	chmod a+x ./nsis/makensis.exe
+ifeq ($(USE_GPL_LIBS), 1)
+	cd $(JULIAHOME)/dist-extras && \
+	$(JLDOWNLOAD) busybox.exe http://frippery.org/files/busybox/busybox-w32-FRP-483-g31277ab.exe && \
 	chmod a+x busybox.exe
+endif
 
 # various statistics about the build that may interest the user
 ifeq ($(USE_SYSTEM_LLVM), 1)

README.arm.md

@@ -1,7 +1,12 @@
 # Julia binaries for ARM
 
 [Nightly builds](https://status.julialang.org/download/linux-arm) are
-available for ARM.
+available for ARMv7-A.
+
+# Hardware requirements
+
+Julia requires at least `armv6` and `vfpv2` instruction sets. It's recommended
+to use at least `armv7-a`. `armv5` or soft float are not supported.
 
 # Building Julia on ARM
 

README.md

@@ -53,7 +53,7 @@ developers may find the notes in [CONTRIBUTING](https://github.com/JuliaLang/jul
 - **FreeBSD**
 - **Windows**
 
-All systems are supported with both x86/64 (64-bit) and x86 (32-bit) architectures. Experimental and early support for [ARM](https://github.com/JuliaLang/julia/blob/master/README.arm.md) is available too.
+All systems are supported with both x86/64 (64-bit) and x86 (32-bit) architectures. Experimental and early support for [ARM](https://github.com/JuliaLang/julia/blob/master/README.arm.md), AARCH64, and POWER (little-endian) is available too.
 
 <a name="Source-Download-and-Compilation"/>
 ## Source Download and Compilation
@@ -235,12 +235,22 @@ When building Julia, or its dependencies, libraries installed by third party pac
 
 ### FreeBSD
 
-On *FreeBSD Release 9.0*, install the `gcc47`, `git`, and `gmake` packages/ports, and compile Julia with the command:
+On *FreeBSD Release 11.0*, install the gfortran, git, cmake, and gmake packages/ports (`pkg install gcc6 gmake git cmake`), and compile Julia with the command:
 
-    $ gmake FC=gfortran47
+    $ echo 'FC=gfortran6' >> Make.user
+    $ gmake
 
 You must use the `gmake` command on FreeBSD instead of `make`.
 
+Note that Julia is community-supported and we have little control over our upstream dependencies, you may still run into issues with dependencies and YMMV. Current known issues include:
+
+ - The x86 arch doesn't support threading due to lack of compiler runtime library support (set `JULIA_THREADS=0`).
+ - libunwind needs a small patch to its tests to compile.
+ - OpenBLAS patches in pkg haven't been upstreamed.
+ - gfortran can't link binaries. Set `FFLAGS=-Wl,-rpath,/usr/local/lib/gcc6` to work around this (upstream bug submitted to FreeBSD pkg maintainers).
+ - System libraries installed by pkg are not on the compiler path by default. You may need to add `LDFLAGS=/usr/local/lib` and `CPPFLAGS=/usr/local/include` to your environment or `Make.user` file to build successfully.
+
+
 ### Windows
 
 In order to build Julia on Windows, see [README.windows](https://github.com/JuliaLang/julia/blob/master/README.windows.md).
@@ -263,6 +273,7 @@ Building Julia requires that the following software be installed:
 - **[patch]**                   — for modifying source code.
 - **[cmake]**                   — needed to build `libgit2`.
 - **[openssl]**                 — needed for HTTPS support in `libgit2` on Linux, install via `apt-get install libssl-dev` or `yum install openssl-devel`.
+- **[pkg-config]**              - needed to build libgit2 correctly, especially for proxy support
 
 Julia uses the following external libraries, which are automatically downloaded (or in a few cases, included in the Julia source repository) and then compiled from source the first time you run `make`:
 
@@ -283,6 +294,7 @@ Julia uses the following external libraries, which are automatically downloaded
 - **[GMP]** (>= 5.0)         — GNU multiple precision arithmetic library, needed for `BigInt` support.
 - **[MPFR]** (>= 3.0)        — GNU multiple precision floating point library, needed for arbitrary precision floating point (`BigFloat`) support.
 - **[libgit2]** (>= 0.23)    — Git linkable library, used by Julia's package manager
+- **[curl]** (>= 7.50)       — libcurl provides download and proxy support for Julia's package manager
 - **[libssh2]** (>= 1.7)     — library for SSH transport, used by libgit2 for packages with SSH remotes
 - **[mbedtls]** (>= 2.2)     — library used for cryptography and transport layer security, used by libssh2
 - **[utf8proc]** (>= 2.0)    — a library for processing UTF-8 encoded Unicode strings
@@ -325,6 +337,7 @@ For a longer overview of Julia's dependencies, see these [slides](https://github
 [openssl]:      https://www.openssl.org
 [libssh2]:      https://www.libssh2.org
 [mbedtls]:      https://tls.mbed.org/
+[pkg-config]:   https://www.freedesktop.org/wiki/Software/pkg-config/
 
 <a name="System-Provided-Libraries">
 ### System Provided Libraries

base/abstractarray.jl

@@ -9,6 +9,7 @@ typealias RangeIndex Union{Int, Range{Int}, AbstractUnitRange{Int}, Colon}
 typealias DimOrInd Union{Integer, AbstractUnitRange}
 typealias IntOrInd Union{Int, AbstractUnitRange}
 typealias DimsOrInds{N} NTuple{N,DimOrInd}
+typealias NeedsShaping Union{Tuple{Integer,Vararg{Integer}}, Tuple{OneTo,Vararg{OneTo}}}
 
 macro _inline_pure_meta()
     Expr(:meta, :inline, :pure)
@@ -51,15 +52,19 @@ size{N}(x, d1::Integer, d2::Integer, dx::Vararg{Integer, N}) = (size(x, d1), siz
 
 Returns the valid range of indices for array `A` along dimension `d`.
 """
-indices{T,N}(A::AbstractArray{T,N}, d) = d <= N ? indices(A)[d] : OneTo(1)
+function indices{T,N}(A::AbstractArray{T,N}, d)
+    @_inline_meta
+    d <= N ? indices(A)[d] : OneTo(1)
+end
+
 """
     indices(A)
 
 Returns the tuple of valid indices for array `A`.
 """
 function indices(A)
     @_inline_meta
-    map(s->OneTo(s), size(A))
+    map(OneTo, size(A))
 end
 
 # Performance optimization: get rid of a branch on `d` in `indices(A,
@@ -177,12 +182,15 @@ function _strides{M,T,N}(out::NTuple{M}, A::AbstractArray{T,N})
 end
 
 function isassigned(a::AbstractArray, i::Int...)
-    # TODO
     try
         a[i...]
         true
-    catch
-        false
+    catch e
+        if isa(e, BoundsError) || isa(e, UndefRefError)
+            return false
+        else
+            rethrow(e)
+        end
     end
 end
 
@@ -413,14 +421,14 @@ different element type it will create a regular `Array` instead:
      2.18425e-314  2.18425e-314  2.18425e-314  2.18425e-314
 
 """
-similar{T}(a::AbstractArray{T})                          = similar(a, T)
-similar(   a::AbstractArray, T::Type)                    = similar(a, T, to_shape(indices(a)))
-similar{T}(a::AbstractArray{T}, dims::Tuple)             = similar(a, T, to_shape(dims))
-similar{T}(a::AbstractArray{T}, dims::DimOrInd...)       = similar(a, T, to_shape(dims))
-similar(   a::AbstractArray, T::Type, dims::DimOrInd...) = similar(a, T, to_shape(dims))
-similar(   a::AbstractArray, T::Type, dims)              = similar(a, T, to_shape(dims))
+similar{T}(a::AbstractArray{T})                             = similar(a, T)
+similar{T}(a::AbstractArray, ::Type{T})                     = similar(a, T, to_shape(indices(a)))
+similar{T}(a::AbstractArray{T}, dims::Tuple)                = similar(a, T, to_shape(dims))
+similar{T}(a::AbstractArray{T}, dims::DimOrInd...)          = similar(a, T, to_shape(dims))
+similar{T}(a::AbstractArray, ::Type{T}, dims::DimOrInd...)  = similar(a, T, to_shape(dims))
+similar{T}(a::AbstractArray, ::Type{T}, dims::NeedsShaping) = similar(a, T, to_shape(dims))
 # similar creates an Array by default
-similar{N}(a::AbstractArray, T::Type, dims::Dims{N})     = Array{T,N}(dims)
+similar{T,N}(a::AbstractArray, ::Type{T}, dims::Dims{N})    = Array{T,N}(dims)
 
 to_shape(::Tuple{}) = ()
 to_shape(dims::Dims) = dims
@@ -709,10 +717,6 @@ of_indices(x, y) = similar(dims->y, oftype(indices(x), indices(y)))
 
 full(x::AbstractArray) = x
 
-map(::Type{Integer},  a::Array) = map!(Integer, similar(a,typeof(Integer(one(eltype(a))))), a)
-map(::Type{Signed},   a::Array) = map!(Signed, similar(a,typeof(Signed(one(eltype(a))))), a)
-map(::Type{Unsigned}, a::Array) = map!(Unsigned, similar(a,typeof(Unsigned(one(eltype(a))))), a)
-
 ## range conversions ##
 
 map{T<:Real}(::Type{T}, r::StepRange) = T(r.start):T(r.step):T(last(r))
@@ -1653,12 +1657,12 @@ end
 
 # These are needed because map(eltype, As) is not inferrable
 promote_eltype_op(::Any) = (@_pure_meta; Bottom)
+promote_eltype_op(op, A) = (@_pure_meta; promote_op(op, eltype(A)))
 promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T))
-promote_eltype_op{T}(op, ::T               ) = (@_pure_meta; promote_op(op, T))
+promote_eltype_op{T}(op, ::AbstractArray{T}, A) = (@_pure_meta; promote_op(op, T, eltype(A)))
+promote_eltype_op{T}(op, A, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, eltype(A), T))
 promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S))
-promote_eltype_op{R,S}(op, ::AbstractArray{R}, ::S) = (@_pure_meta; promote_op(op, R, S))
-promote_eltype_op{R,S}(op, ::R, ::AbstractArray{S}) = (@_pure_meta; promote_op(op, R, S))
-promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_op(op, eltype(A), promote_eltype_op(op, B, C, D...)))
+promote_eltype_op(op, A, B, C, D...) = (@_pure_meta; promote_eltype_op(op, promote_eltype_op(op, A, B), C, D...))
 
 ## 1 argument
 map!{F}(f::F, A::AbstractArray) = map!(f, A, A)

base/array.jl

@@ -126,18 +126,18 @@ similar{N}(a::Array, T::Type, dims::Dims{N}) = Array{T,N}(dims)
 similar{T,N}(a::Array{T}, dims::Dims{N})     = Array{T,N}(dims)
 
 # T[x...] constructs Array{T,1}
-function getindex(T::Type, vals...)
+function getindex{T}(::Type{T}, vals...)
     a = Array{T,1}(length(vals))
     @inbounds for i = 1:length(vals)
         a[i] = vals[i]
     end
     return a
 end
 
-getindex(T::Type) = Array{T,1}(0)
-getindex(T::Type, x) = (a = Array{T,1}(1); @inbounds a[1] = x; a)
-getindex(T::Type, x, y) = (a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a)
-getindex(T::Type, x, y, z) = (a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a)
+getindex{T}(::Type{T}) = (@_inline_meta; Array{T,1}(0))
+getindex{T}(::Type{T}, x) = (@_inline_meta; a = Array{T,1}(1); @inbounds a[1] = x; a)
+getindex{T}(::Type{T}, x, y) = (@_inline_meta; a = Array{T,1}(2); @inbounds (a[1] = x; a[2] = y); a)
+getindex{T}(::Type{T}, x, y, z) = (@_inline_meta; a = Array{T,1}(3); @inbounds (a[1] = x; a[2] = y; a[3] = z); a)
 
 function getindex(::Type{Any}, vals::ANY...)
     a = Array{Any,1}(length(vals))
@@ -288,8 +288,8 @@ else
     _default_eltype(itr::ANY) = Any
 end
 
-_array_for(T, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer))
-_array_for(T, itr, ::HasShape) = similar(Array{T}, indices(itr))
+_array_for{T}(::Type{T}, itr, ::HasLength) = Array{T,1}(Int(length(itr)::Integer))
+_array_for{T}(::Type{T}, itr, ::HasShape) = similar(Array{T}, indices(itr))
 
 function collect(itr::Generator)
     isz = iteratorsize(itr.iter)
@@ -1077,15 +1077,18 @@ function find(testf::Function, A)
     # use a dynamic-length array to store the indexes, then copy to a non-padded
     # array for the return
     tmpI = Array{Int,1}(0)
+    inds = _index_remapper(A)
     for (i,a) = enumerate(A)
         if testf(a)
-            push!(tmpI, i)
+            push!(tmpI, inds[i])
         end
     end
     I = Array{Int,1}(length(tmpI))
     copy!(I, tmpI)
     return I
 end
+_index_remapper(A::AbstractArray) = linearindices(A)
+_index_remapper(iter) = Colon()  # safe for objects that don't implement length
 
 """
     find(A)
@@ -1110,9 +1113,10 @@ function find(A)
     nnzA = countnz(A)
     I = Vector{Int}(nnzA)
     count = 1
+    inds = _index_remapper(A)
     for (i,a) in enumerate(A)
         if a != 0
-            I[count] = i
+            I[count] = inds[i]
             count += 1
         end
     end