Merge pull request #3603 from timholy/mmap

RFC: support mmap on Windows

base/deprecated.jl

@@ -216,6 +216,8 @@ export PipeString
 @deprecate  ~(s::IntSet)                 complement(s)
 @deprecate openblas_set_num_threads      blas_set_num_threads
 @deprecate check_openblas                check_blas
+@deprecate msync(A::Array, flags::Int)    msync(A)
+@deprecate msync(A::BitArray, flags::Int) msync(A)
 
 deprecated_ls() = run(`ls -l`)
 deprecated_ls(args::Cmd) = run(`ls -l $args`)

base/exports.jl

@@ -1041,6 +1041,7 @@ export
     fd,
     fdio,
     FDWatcher,
+    isreadonly,
     UV_READABLE,
     UV_WRITEABLE,
     flush,
@@ -1056,7 +1057,6 @@ export
     mmap_grow,
     mmap_stream_settings,
     msync,
-    munmap,
     nb_available,
     open,
     open_any_tcp_port,

base/io.jl

@@ -234,6 +234,7 @@ show(io::IO, s::IOStream) = print(io, "IOStream(", s.name, ")")
 fd(s::IOStream) = int(ccall(:jl_ios_fd, Clong, (Ptr{Void},), s.ios))
 close(s::IOStream) = ccall(:ios_close, Void, (Ptr{Void},), s.ios)
 flush(s::IOStream) = ccall(:ios_flush, Void, (Ptr{Void},), s.ios)
+isreadonly(s::IOStream) = bool(ccall(:ios_get_readonly, Cint, (Ptr{Void},), s.ios))
 
 truncate(s::IOStream, n::Integer) =
     (ccall(:ios_trunc, Int32, (Ptr{Void}, Uint), s.ios, n)==0 ||

base/mmap.jl

@@ -1,3 +1,21 @@
+### Generic interface ###
+
+# Arrays
+mmap_array{T,N}(::Type{T}, dims::NTuple{N,Int}, s::IO) = mmap_array(T, dims, s, position(s))
+
+msync{T}(A::Array{T}) = msync(pointer(A), length(A)*sizeof(T))
+
+# BitArrays
+mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Int}, s::IOStream, offset::FileOffset) =
+    mmap_bitarray(dims, s, offset)
+mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Int}, s::IOStream) = mmap_bitarray(dims, s, position(s))
+mmap_bitarray{N}(dims::NTuple{N,Int}, s::IOStream) = mmap_bitarray(dims, s, position(s))
+
+msync(B::BitArray) = msync(pointer(B.chunks), length(B.chunks)*sizeof(Uint64))
+
+### UNIX implementation ###
+
+@unix_only begin
 # Low-level routines
 # These are needed for things like MAP_ANONYMOUS
 function mmap(len::Integer, prot::Integer, flags::Integer, fd::Integer, offset::FileOffset)
@@ -10,8 +28,8 @@ function mmap(len::Integer, prot::Integer, flags::Integer, fd::Integer, offset::
     offset_page::FileOffset = ifloor(offset/pagesize)*pagesize
     len_page::Int = (offset-offset_page) + len
     # Mmap the file
-    p = ccall(:mmap, Ptr{Void}, (Ptr{Void}, Int, Int32, Int32, Int32, FileOffset), C_NULL, len_page, prot, flags, fd, offset_page)
-    if convert(Int,p) == -1
+    p = ccall(:mmap, Ptr{Void}, (Ptr{Void}, Csize_t, Cint, Cint, Cint, FileOffset), C_NULL, len_page, prot, flags, fd, offset_page)
+    if int(p) == -1
         error("Memory mapping failed", strerror())
     end
     # Also return a pointer that compensates for any adjustment in the offset
@@ -24,30 +42,30 @@ end
 # Note: a few mappable streams do not support lseek. When Julia
 # supports structures in ccall, switch to fstat.
 function mmap_grow(len::Integer, prot::Integer, flags::Integer, fd::Integer, offset::FileOffset)
-    const SEEK_SET::Int32 = 0
-    const SEEK_CUR::Int32 = 1
-    const SEEK_END::Int32 = 2
+    const SEEK_SET::Cint = 0
+    const SEEK_CUR::Cint = 1
+    const SEEK_END::Cint = 2
     # Save current file position so we can restore it later
-    cpos = ccall(:lseek, FileOffset, (Int32, FileOffset, Int32), fd, 0, SEEK_CUR)
+    cpos = ccall(:lseek, FileOffset, (Cint, FileOffset, Cint), fd, 0, SEEK_CUR)
     if cpos < 0
         error(strerror())
     end
-    filelen = ccall(:lseek, FileOffset, (Int32, FileOffset, Int32), fd, 0, SEEK_END)
+    filelen = ccall(:lseek, FileOffset, (Cint, FileOffset, Cint), fd, 0, SEEK_END)
     if filelen < 0
         error(strerror())
     end
     if (filelen < offset + len)
-        n = ccall(:pwrite, Int, (Int32, Ptr{Void}, Uint, FileOffset), fd, int8([0]), 1, offset + len - 1)
+        n = ccall(:pwrite, Cssize_t, (Cint, Ptr{Void}, Uint, FileOffset), fd, int8([0]), 1, offset + len - 1)
         if (n < 1)
             error(strerror())
         end
     end
-    cpos = ccall(:lseek, FileOffset, (Int32, FileOffset, Int32), fd, cpos, SEEK_SET)
+    cpos = ccall(:lseek, FileOffset, (Cint, FileOffset, Cint), fd, cpos, SEEK_SET)
     return mmap(len, prot, flags, fd, offset)
 end
 
 function munmap(p::Ptr,len::Integer)
-    ret = ccall(:munmap,Int32,(Ptr{Void},Int),p,len)
+    ret = ccall(:munmap,Cint,(Ptr{Void},Int),p,len)
     if ret != 0
         error(strerror())
     end
@@ -57,21 +75,23 @@ const MS_ASYNC = 1
 const MS_INVALIDATE = 2
 const MS_SYNC = 4
 function msync(p::Ptr, len::Integer, flags::Integer)
-    ret = ccall(:msync, Int32, (Ptr{Void}, Int, Int32), p, len, flags)
+    ret = ccall(:msync, Cint, (Ptr{Void}, Csize_t, Cint), p, len, flags)
     if ret != 0
         error(strerror())
     end
-end    
+end
+msync(p::Ptr, len::Integer) = msync(p, len, MS_SYNC)
 
 # Higher-level functions
 # Determine a stream's read/write mode, and return prot & flags
 # appropriate for mmap
+# We could use isreadonly here, but it's worth checking that it's readable too
 function mmap_stream_settings(s::IO)
-    const PROT_READ::Int32 = 1
-    const PROT_WRITE::Int32 = 2
-    const MAP_SHARED::Int32 = 1
-    const F_GETFL::Int32 = 3
-    mode = ccall(:fcntl,Int32,(Int32,Int32),fd(s),F_GETFL)
+    const PROT_READ::Cint = 1
+    const PROT_WRITE::Cint = 2
+    const MAP_SHARED::Cint = 1
+    const F_GETFL::Cint = 3
+    mode = ccall(:fcntl,Cint,(Cint,Cint),fd(s),F_GETFL)
     mode = mode & 3
     if mode == 0
         prot = PROT_READ
@@ -100,16 +120,6 @@ function mmap_array{T,N}(::Type{T}, dims::NTuple{N,Int}, s::IO, offset::FileOffs
     finalizer(A,x->munmap(pmap,len+delta))
     return A
 end
-mmap_array{T,N}(::Type{T}, dims::NTuple{N,Int}, s::IO) = mmap_array(T, dims, s, position(s))
-msync{T}(A::Array{T}, flags::Int) = msync(pointer(A), prod(size(A))*sizeof(T), flags)
-msync{T}(A::Array{T}) = msync(A,MS_SYNC)
-
-# For storing information about a mmapped-array
-type MmapArrayInfo
-    pathname::ByteString
-    eltype::Type
-    dims::Dims
-end
 
 # Mmapped-bitarray constructor
 function mmap_bitarray{N}(dims::NTuple{N,Int}, s::IOStream, offset::FileOffset)
@@ -133,12 +143,47 @@ function mmap_bitarray{N}(dims::NTuple{N,Int}, s::IOStream, offset::FileOffset)
     if N != 1
         B.dims = Int[i for i in dims]
     end
+    finalizer(B, x->munmap(pointer(B.chunks), length(B.chunks)*sizeof(Uint64)))
     return B
 end
-mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Int}, s::IOStream, offset::FileOffset) =
-    mmap_bitarray(dims, s, offset)
-mmap_bitarray{N}(::Type{Bool}, dims::NTuple{N,Int}, s::IOStream) = mmap_bitarray(dims, s, position(s))
-mmap_bitarray{N}(dims::NTuple{N,Int}, s::IOStream) = mmap_bitarray(dims, s, position(s))
+end
+
 
-msync(B::BitArray, flags::Integer) = msync(pointer(B.chunks), flags)
-msync(B::BitArray) = msync(B.chunks,MS_SYNC)
+### Windows implementation ###
+@windows_only begin
+# Mmapped-array constructor
+function mmap_array{T,N}(::Type{T}, dims::NTuple{N,Int}, s::IO, offset::FileOffset)
+    shandle = _get_osfhandle(RawFD(fd(s)))
+    ro = isreadonly(shandle)
+    flprotect = ro ? 0x02 : 0x04
+    szarray = convert(Csize_t, prod(dims))*sizeof(T)
+    szfile = szarray + convert(Csize_t, offset)
+    mmaphandle = ccall(:CreateFileMappingA, stdcall, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Cint, Cint, Cint, Ptr{Void}), shandle.handle, C_NULL, flprotect, szfile>>32, szfile&0xffffffff, C_NULL)
+    if mmaphandle == C_NULL
+        error("Could not create file mapping")
+    end
+    access = ro ? 4 : 2
+    viewhandle = ccall(:MapViewOfFile, stdcall, Ptr{Void}, (Ptr{Void}, Cint, Cint, Cint, Csize_t), mmaphandle, access, offset>>32, offset&0xffffffff, szarray)
+    if viewhandle == C_NULL
+        error("Could not create mapping view")
+    end
+    A = pointer_to_array(pointer(T, viewhandle), dims)
+    finalizer(A, x->munmap(viewhandle))
+    return A
+end
+
+function munmap(viewhandle::Ptr)
+    status = bool(ccall(:UnmapViewOfFile, stdcall, Cint, (Ptr{Void},), viewhandle))
+    if !status
+        error("Could not unmap view")
+    end
+end
+
+function msync(p::Ptr, len::Integer)
+    status = bool(ccall(:FlushViewOfFile, stdcall, Cint, (Ptr{Void}, Csize_t), p, len))
+    if !status
+        error("Could not msync")
+    end
+end
+
+end

base/poll.jl

@@ -57,7 +57,15 @@ end
 @windows_only typealias FDW_FD WindowsRawSocket
 
 @unix_only _get_osfhandle(fd::RawFD) = fd
-@windows_only _get_osfhandle(fd::RawFD) = WindowsRawSocket(ccall(:_get_osfhandle,Ptr{Void},(Int32,),fd.fd))
+@windows_only begin
+function _get_osfhandle(fd::RawFD)
+    handle = ccall(:_get_osfhandle,Ptr{Void},(Int32,),fd.fd)
+    if int(handle) == -1
+        error("Could not get handle")
+    end
+    WindowsRawSocket(handle)
+end
+end
 
 type FDWatcher <: UVPollingWatcher
     handle::Ptr{Void}

src/julia.expmap

@@ -19,6 +19,7 @@
     ios_fd;
     ios_file;
     ios_flush;
+    ios_get_readonly;
     ios_getc;
     ios_getutf8;
     ios_mem;

src/support/ios.c

@@ -677,6 +677,11 @@ int ios_bufmode(ios_t *s, bufmode_t mode)
     return 0;
 }
 
+int ios_get_readonly(ios_t *s)
+{
+    return s->readonly;
+}
+
 void ios_set_readonly(ios_t *s)
 {
     if (s->readonly) return;

src/support/ios.h

@@ -76,6 +76,7 @@ DLLEXPORT char *ios_takebuf(ios_t *s, size_t *psize);  // release buffer to call
 // set buffer space to use
 DLLEXPORT int ios_setbuf(ios_t *s, char *buf, size_t size, int own);
 DLLEXPORT int ios_bufmode(ios_t *s, bufmode_t mode);
+DLLEXPORT int ios_get_readonly(ios_t *s);
 DLLEXPORT void ios_set_readonly(ios_t *s);
 DLLEXPORT size_t ios_copy(ios_t *to, ios_t *from, size_t nbytes);
 DLLEXPORT size_t ios_copyall(ios_t *to, ios_t *from);

test/file.jl

@@ -101,7 +101,28 @@ test_touch(1)
 test_monitor(1)
 test_monitor(0.1)
 
-
+##########
+#  mmap  #
+##########
+
+s = open(file, "w")
+write(s, "Hello World\n")
+close(s)
+s = open(file, "r")
+@test isreadonly(s) == true
+c = mmap_array(Uint8, (11,), s)
+@test c == "Hello World".data
+close(s)
+s = open(file, "r+")
+@test isreadonly(s) == false
+c = mmap_array(Uint8, (11,), s)
+c[5] = uint8('x')
+msync(c)
+close(s)
+s = open(file, "r")
+str = readline(s)
+close(s)
+@test beginswith(str, "Hellx World")
 
 #######################################################################
 # This section tests temporary file and directory creation.           #