Skip to content

Commit d188fe3

Browse files
authored
Basic filesystems support for Wasm: support WASI-based filesystem for wasmWasi (#257)
Implemented basic filesystem support on top of Wasm WASI.
1 parent 7b23cf9 commit d188fe3

File tree

11 files changed

+1333
-48
lines changed

11 files changed

+1333
-48
lines changed

core/build.gradle.kts

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
*/
55

66
import org.jetbrains.dokka.gradle.DokkaTaskPartial
7-
import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinTargetWithNodeJsDsl
87

98
plugins {
109
id("kotlinx-io-multiplatform")
@@ -32,22 +31,6 @@ kotlin {
3231
}
3332
}
3433

35-
fun KotlinTargetWithNodeJsDsl.filterSmokeTests() {
36-
this.nodejs {
37-
testTask(Action {
38-
useMocha {
39-
timeout = "300s"
40-
}
41-
filter.setExcludePatterns("*SmokeFileTest*")
42-
})
43-
}
44-
}
45-
46-
@OptIn(org.jetbrains.kotlin.gradle.targets.js.dsl.ExperimentalWasmDsl::class)
47-
wasmWasi {
48-
filterSmokeTests()
49-
}
50-
5134
sourceSets {
5235
commonMain {
5336
dependencies {
@@ -76,6 +59,32 @@ tasks.withType<DokkaTaskPartial>().configureEach {
7659
}
7760
}
7861

62+
tasks.named("wasmWasiNodeTest") {
63+
// TODO: remove once https://youtrack.jetbrains.com/issue/KT-65179 solved
64+
doFirst {
65+
val layout = project.layout
66+
val templateFile = layout.projectDirectory.file("wasmWasi/test/test-driver.mjs.template").asFile
67+
68+
val driverFile = layout.buildDirectory.file(
69+
"compileSync/wasmWasi/test/testDevelopmentExecutable/kotlin/kotlinx-io-kotlinx-io-core-wasm-wasi-test.mjs"
70+
)
71+
72+
fun File.mkdirsAndEscape(): String {
73+
mkdirs()
74+
return absolutePath.replace("\\", "\\\\")
75+
}
76+
77+
val tmpDir = temporaryDir.resolve("kotlinx-io-core-wasi-test").mkdirsAndEscape()
78+
val tmpDir2 = temporaryDir.resolve("kotlinx-io-core-wasi-test-2").mkdirsAndEscape()
79+
80+
val newDriver = templateFile.readText()
81+
.replace("<SYSTEM_TEMP_DIR>", tmpDir, false)
82+
.replace("<SYSTEM_TEMP_DIR2>", tmpDir2, false)
83+
84+
driverFile.get().asFile.writeText(newDriver)
85+
}
86+
}
87+
7988
animalsniffer {
8089
annotation = "kotlinx.io.files.AnimalSnifferIgnore"
8190
}

core/common/src/files/FileSystem.kt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ public sealed interface FileSystem {
7474
* filesystems (or different volumes, on Windows) and the operation could not be performed atomically,
7575
* [UnsupportedOperationException] is thrown.
7676
*
77+
* On some platforms, like Wasm-WASI, there is no way to tell if the underlying filesystem supports atomic move.
78+
* In such cases, the move will be performed and no [UnsupportedOperationException] will be thrown.
79+
*
7780
* When [destination] is an existing directory, the operation may fail on some platforms
7881
* (on Windows, particularly).
7982
*

core/common/test/files/SmokeFileTest.kt

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ class SmokeFileTest {
3232
return f
3333
}
3434

35+
private fun removeOnExit(path: Path) {
36+
files.add(path)
37+
}
38+
3539
@OptIn(ExperimentalStdlibApi::class)
3640
@Test
3741
fun readWriteFile() {
@@ -46,6 +50,22 @@ class SmokeFileTest {
4650
}
4751
}
4852

53+
@OptIn(ExperimentalStdlibApi::class)
54+
@Test
55+
fun writeFlush() {
56+
val path = createTempPath()
57+
SystemFileSystem.sink(path).buffered().use {
58+
it.writeString("hello")
59+
it.flush()
60+
it.writeString(" world")
61+
it.flush()
62+
}
63+
64+
SystemFileSystem.source(path).buffered().use {
65+
assertEquals("hello world", it.readLine())
66+
}
67+
}
68+
4969
@OptIn(ExperimentalStdlibApi::class)
5070
@Test
5171
fun readNotExistingFile() {
@@ -362,11 +382,17 @@ class SmokeFileTest {
362382
}
363383

364384
val cwd = SystemFileSystem.resolve(Path("."))
365-
val parentRel = Path("..")
366-
assertEquals(cwd.parent, SystemFileSystem.resolve(parentRel))
367385

368-
assertEquals(cwd, SystemFileSystem.resolve(cwd),
369-
"Absolute path resolution should not alter the path")
386+
SystemFileSystem.createDirectories(Path("a"))
387+
removeOnExit(Path("a"))
388+
389+
val childRel = Path("a", "..")
390+
assertEquals(cwd, SystemFileSystem.resolve(childRel))
391+
392+
assertEquals(
393+
cwd, SystemFileSystem.resolve(cwd),
394+
"Absolute path resolution should not alter the path"
395+
)
370396

371397
// root
372398
// |-> a -> b

core/wasmWasi/src/-WasmUtils.kt

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors.
3+
* Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file.
4+
*/
5+
@file:OptIn(UnsafeWasmMemoryApi::class)
6+
7+
package kotlinx.io
8+
9+
import kotlin.wasm.unsafe.MemoryAllocator
10+
import kotlin.wasm.unsafe.Pointer
11+
import kotlin.wasm.unsafe.UnsafeWasmMemoryApi
12+
13+
internal fun Pointer.loadInt(offset: Int): Int = (this + offset).loadInt()
14+
internal fun Pointer.loadLong(offset: Int): Long = (this + offset).loadLong()
15+
internal fun Pointer.loadShort(offset: Int): Short = (this + offset).loadShort()
16+
internal fun Pointer.loadByte(offset: Int): Byte = (this + offset).loadByte()
17+
18+
internal fun Pointer.loadBytes(length: Int): ByteArray {
19+
val buffer = ByteArray(length)
20+
for (offset in 0 until length) {
21+
buffer[offset] = this.loadByte(offset)
22+
}
23+
return buffer
24+
}
25+
26+
internal fun Pointer.storeInt(offset: Int, value: Int): Unit = (this + offset).storeInt(value)
27+
internal fun Pointer.storeLong(offset: Int, value: Long): Unit = (this + offset).storeLong(value)
28+
internal fun Pointer.storeShort(offset: Int, value: Short): Unit = (this + offset).storeShort(value)
29+
internal fun Pointer.storeByte(offset: Int, value: Byte): Unit = (this + offset).storeByte(value)
30+
31+
internal fun Pointer.storeBytes(bytes: ByteArray) {
32+
for (offset in bytes.indices) {
33+
this.storeByte(offset, bytes[offset])
34+
}
35+
}
36+
37+
@OptIn(UnsafeWasmMemoryApi::class)
38+
internal fun Buffer.readToLinearMemory(pointer: Pointer, bytes: Int) {
39+
checkBounds(size, 0L, bytes.toLong())
40+
var current: Segment? = head
41+
var remaining = bytes
42+
var currentPtr = pointer
43+
do {
44+
current!!
45+
val data = current.data
46+
val pos = current.pos
47+
val limit = current.limit
48+
val read = minOf(remaining, limit - pos)
49+
for (offset in 0 until read) {
50+
currentPtr.storeByte(offset, data[pos + offset])
51+
}
52+
currentPtr += read
53+
remaining -= read
54+
current = current.next
55+
} while (current != head && remaining > 0)
56+
check(remaining == 0)
57+
skip(bytes.toLong())
58+
}
59+
60+
61+
internal fun Buffer.writeFromLinearMemory(pointer: Pointer, bytes: Int) {
62+
var remaining = bytes
63+
var currentPtr = pointer
64+
while (remaining > 0) {
65+
val segment = writableSegment(1)
66+
val limit = segment.limit
67+
val data = segment.data
68+
val toWrite = minOf(data.size - limit, remaining)
69+
70+
for (offset in 0 until toWrite) {
71+
data[limit + offset] = currentPtr.loadByte(offset)
72+
}
73+
74+
currentPtr += toWrite
75+
remaining -= toWrite
76+
segment.limit += toWrite
77+
size += toWrite
78+
}
79+
}
80+
81+
/**
82+
* Encoding [value] into a NULL-terminated byte sequence using UTF-8 encoding
83+
* and writes it to a memory region allocated to fit the sequence.
84+
* Return a pointer to the beginning of the written byte sequence and its length.
85+
*/
86+
@OptIn(UnsafeWasmMemoryApi::class)
87+
internal fun MemoryAllocator.storeString(value: String): Pair<Pointer, Int> {
88+
val bytes = value.encodeToByteArray()
89+
val ptr = allocate(bytes.size + 1)
90+
ptr.storeBytes(bytes)
91+
ptr.storeByte(bytes.size, 0)
92+
return ptr to (bytes.size + 1)
93+
}
94+
95+
/**
96+
* Encodes [value] into a NULL-terminated byte sequence using UTF-8 encoding,
97+
* stores it in memory starting at the position this pointer points to,
98+
* and returns the length of the stored bytes sequence.
99+
*/
100+
internal fun Pointer.allocateString(value: String): Int {
101+
val bytes = value.encodeToByteArray()
102+
storeBytes(bytes)
103+
storeByte(bytes.size, 0)
104+
return bytes.size + 1
105+
}
106+
107+
/**
108+
* Allocates memory to hold a single integer value.
109+
*/
110+
@UnsafeWasmMemoryApi
111+
internal fun MemoryAllocator.allocateInt(): Pointer = allocate(Int.SIZE_BYTES)

0 commit comments

Comments
 (0)