Skip to content

Commit 0b5448a

Browse files
jartjart
committed
Implement system polyfill for win32 / posix.1
I don't have access to Microsoft Visual Studio right now (aside from the the Github Actions CI system) but I think this code should come close to what we want in terms of polyfilling UNIX functionality.
1 parent 5b8023d commit 0b5448a

File tree

2 files changed

+264
-22
lines changed

2 files changed

+264
-22
lines changed

‎main.cpp

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#include "ggml.h"
22

33
#include "utils.h"
4+
#include "mmap.h"
45

56
#include <cassert>
6-
#include <cerrno>
77
#include <cmath>
88
#include <cstdio>
99
#include <cstring>
@@ -207,27 +207,6 @@ void *memalign(size_t a, size_t n) {
207207
return p;
208208
}
209209

210-
int posix_memalign(void **pp, size_t a, size_t n) {
211-
int e;
212-
void *m;
213-
size_t q, r;
214-
q = a / sizeof(void *);
215-
r = a % sizeof(void *);
216-
if (!r && q && IS2POW(q)) {
217-
e = errno;
218-
m = memalign(a, n);
219-
if (m) {
220-
*pp = m;
221-
return 0;
222-
} else {
223-
errno = e;
224-
return ENOMEM;
225-
}
226-
} else {
227-
return EINVAL;
228-
}
229-
}
230-
231210
void *malloc(size_t n) {
232211
return memalign(MAGIC_ALGN, n);
233212
}

‎mmap.h

Lines changed: 263 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,263 @@
1+
#pragma once
2+
3+
// portable mmap() implementation
4+
//
5+
// - supports win32 (needs vista+)
6+
// - supports posix.1 (no map_anonymous)
7+
//
8+
// notes on windows
9+
//
10+
// - no errno support
11+
// - not designed to support mprotect()
12+
// - very poor support for memory intervals
13+
14+
#include <fcntl.h>
15+
#include <stddef.h>
16+
#include <stdint.h>
17+
#include <stdlib.h>
18+
#include <sys/stat.h>
19+
20+
#if defined(_MSC_VER) || defined(__MINGW32__)
21+
#ifndef __MINGW32__
22+
#include <Windows.h>
23+
#else
24+
#include <windows.h>
25+
#endif
26+
#include <io.h>
27+
#include <atomic>
28+
#include <map>
29+
30+
#ifndef PROT_READ
31+
#define PROT_READ 1
32+
#endif
33+
#ifndef PROT_WRITE
34+
#define PROT_WRITE 2
35+
#endif
36+
#ifndef PROT_EXEC
37+
#define PROT_EXEC 4
38+
#endif
39+
40+
#ifndef MAP_SHARED
41+
#define MAP_SHARED 1
42+
#endif
43+
#ifndef MAP_PRIVATE
44+
#define MAP_PRIVATE 2
45+
#endif
46+
#ifndef MAP_FIXED
47+
#define MAP_FIXED 16
48+
#endif
49+
#ifndef MAP_ANONYMOUS
50+
#define MAP_ANONYMOUS 32
51+
#endif
52+
#ifndef MAP_FAILED
53+
#define MAP_FAILED ((void*)-1)
54+
#endif
55+
56+
#ifndef O_RDONLY
57+
#define O_RDONLY _O_RDWR // intentional smudge for mmap()
58+
#endif
59+
#ifndef O_WRONLY
60+
#define O_WRONLY _O_WRONLY
61+
#endif
62+
#ifndef O_RDWR
63+
#define O_RDWR _O_RDWR
64+
#endif
65+
#ifndef O_CREAT
66+
#define O_CREAT _O_CREAT
67+
#endif
68+
#ifndef O_TRUNC
69+
#define O_TRUNC _O_TRUNC
70+
#endif
71+
#ifndef O_EXCL
72+
#define O_EXCL _O_EXCL
73+
#endif
74+
75+
#ifndef MADV_NORMAL
76+
#define MADV_NORMAL 0
77+
#endif
78+
#ifndef MADV_DONTNEED
79+
#define MADV_DONTNEED 4
80+
#endif
81+
#ifndef MADV_RANDOM
82+
#define MADV_RANDOM 1
83+
#endif
84+
#ifndef MADV_SEQUENTIAL
85+
#define MADV_SEQUENTIAL 2
86+
#endif
87+
#ifndef MADV_WILLNEED
88+
#define MADV_WILLNEED 3
89+
#endif
90+
91+
#ifndef mmap
92+
#define mmap WinMap
93+
#endif
94+
#ifndef munmap
95+
#define munmap WinUnmap
96+
#endif
97+
#ifndef open
98+
#define open _open
99+
#endif
100+
#ifndef close
101+
#define close _close
102+
#endif
103+
#ifndef fstat
104+
#define fstat _fstat
105+
#endif
106+
#ifndef madvise
107+
#define madvise WinMadvise
108+
#endif
109+
#ifndef ftruncate
110+
#define ftruncate WinFtruncate
111+
#endif
112+
113+
static std::atomic<unsigned> g_winlock;
114+
static std::map<LPVOID, HANDLE> g_winmap;
115+
116+
static void WinLock(void) {
117+
while (!g_winlock.exchange(1, std::memory_order_acquire));
118+
}
119+
120+
static void WunLock(void) {
121+
g_winlock.store(0, std::memory_order_release);
122+
}
123+
124+
static int WinMadvise(int fd, size_t length, int flags) {
125+
return 0;
126+
}
127+
128+
static int WinFtruncate(int fd, uint64_t length) {
129+
return _chsize_s(fd, length) ? -1 : 0;
130+
}
131+
132+
static int WinUnmap(void *addr, size_t length) {
133+
HANDLE hand;
134+
WinLock();
135+
hand = g_winmap[addr];
136+
g_winmap[addr] = 0;
137+
WunLock();
138+
if (hand) {
139+
UnmapViewOfFile(addr);
140+
CloseHandle(hand);
141+
return 0;
142+
} else {
143+
return -1;
144+
}
145+
}
146+
147+
static void *WinMap(void *addr, size_t length, int prot,
148+
int flags, int fd, uint64_t offset) {
149+
HANDLE hFile;
150+
DWORD winprot;
151+
DWORD access = 0;
152+
HANDLE hand = NULL;
153+
LPVOID res = NULL;
154+
if (prot & PROT_READ) {
155+
access |= FILE_MAP_READ;
156+
}
157+
if (prot & PROT_WRITE) {
158+
access |= FILE_MAP_WRITE;
159+
}
160+
if (prot & PROT_EXEC) {
161+
access |= FILE_MAP_EXECUTE;
162+
}
163+
if (flags & MAP_PRIVATE) {
164+
// private mapping
165+
if (prot & PROT_EXEC) {
166+
if (prot & PROT_WRITE) {
167+
if (flags & MAP_ANONYMOUS) {
168+
winprot = PAGE_EXECUTE_READWRITE;
169+
} else {
170+
winprot = PAGE_EXECUTE_WRITECOPY;
171+
}
172+
} else {
173+
winprot = PAGE_EXECUTE_READ;
174+
}
175+
} else if (prot & PROT_WRITE) {
176+
if (flags & MAP_ANONYMOUS) {
177+
winprot = PAGE_READWRITE;
178+
} else {
179+
winprot = PAGE_WRITECOPY;
180+
}
181+
} else {
182+
winprot = PAGE_READONLY;
183+
}
184+
} else {
185+
// shared mapping
186+
if (prot & PROT_EXEC) {
187+
if (prot & PROT_WRITE) {
188+
winprot = PAGE_EXECUTE_READWRITE;
189+
} else {
190+
winprot = PAGE_EXECUTE_READ;
191+
}
192+
} else if (prot & PROT_WRITE) {
193+
winprot = PAGE_READWRITE;
194+
} else {
195+
winprot = PAGE_READONLY;
196+
}
197+
}
198+
if (flags & MAP_ANONYMOUS) {
199+
hFile = INVALID_HANDLE_VALUE;
200+
offset = 0;
201+
} else {
202+
hFile = (HANDLE)_get_osfhandle(fd);
203+
if (hFile == INVALID_HANDLE_VALUE) {
204+
return MAP_FAILED;
205+
}
206+
}
207+
if (flags & MAP_FIXED) {
208+
if (!addr) {
209+
return MAP_FAILED;
210+
} else {
211+
WinUnmap(addr, length);
212+
}
213+
}
214+
hand = CreateFileMapping(hFile, 0, winprot,
215+
(offset + length) >> 32,
216+
(offset + length), 0);
217+
if (!hand) {
218+
return MAP_FAILED;
219+
}
220+
res = MapViewOfFileEx(hand, access, offset >> 32,
221+
offset, length, addr);
222+
if (!res) {
223+
CloseHandle(hand);
224+
return MAP_FAILED;
225+
}
226+
WinLock();
227+
g_winmap[res] = hand;
228+
WunLock();
229+
return res;
230+
}
231+
232+
#else
233+
#include <unistd.h>
234+
#include <sys/mman.h>
235+
#ifndef MAP_ANONYMOUS
236+
#define MAP_ANONYMOUS 0x10000000
237+
238+
static void *PosixMmap(void *addr, size_t length, int prot,
239+
int flags, int fd, uint64_t offset) {
240+
int tfd;
241+
void *res;
242+
char path[] = "/tmp/llama.dat.XXXXXX";
243+
if (~flags & MAP_ANONYMOUS) {
244+
res = mmap(addr, length, prot, flags, fd, offset);
245+
} else if ((tfd = mkstemp(path)) != -1) {
246+
unlink(path);
247+
if (!ftruncate(tfd, length)) {
248+
res = mmap(addr, length, prot, flags & ~MAP_ANONYMOUS, tfd, 0);
249+
} else {
250+
res = MAP_FAILED;
251+
}
252+
close(tfd);
253+
} else {
254+
res = MAP_FAILED;
255+
}
256+
return res;
257+
}
258+
259+
#ifndef mmap
260+
#define mmap PosixMmap
261+
#endif
262+
#endif // MAP_ANONYMOUS
263+
#endif // _MSC_VER

0 commit comments

Comments
 (0)