Merge pull request #167 from Kotlin/develop

v0.2.2

Author: devcrocod

README.md

@@ -28,8 +28,8 @@ repositories {
 }
 
 dependencies {
-    implementation "org.jetbrains.kotlinx:multik-core:0.2.0"
-    implementation "org.jetbrains.kotlinx:multik-default:0.2.0"
+    implementation "org.jetbrains.kotlinx:multik-core:0.2.2"
+    implementation "org.jetbrains.kotlinx:multik-default:0.2.2"
 }
 ```
 
@@ -40,8 +40,8 @@ repositories {
 }
 
 dependencies {
-    implementation("org.jetbrains.kotlinx:multik-core:0.2.0")
-    implementation("org.jetbrains.kotlinx:multik-default:0.2.0")
+    implementation("org.jetbrains.kotlinx:multik-core:0.2.2")
+    implementation("org.jetbrains.kotlinx:multik-default:0.2.2")
 }
 ```
 
@@ -52,7 +52,7 @@ kotlin {
     sourceSets {
         val commonMain by getting {
             dependencies {
-                implementation("org.jetbrains.kotlinx:multik-core:0.2.0")
+                implementation("org.jetbrains.kotlinx:multik-core:0.2.2")
             }
         }
     }
@@ -66,7 +66,7 @@ kotlin {
     sourceSets {
         val jvmName by getting {
             dependencies {
-                implementation("org.jetbrains.kotlinx:multik-core-jvm:0.2.0")
+                implementation("org.jetbrains.kotlinx:multik-core-jvm:0.2.2")
             }
         }
     }

gradle.properties

@@ -1,15 +1,15 @@
 kotlin.code.style=official
-multik_version=0.2.1
+multik_version=0.2.2
 
 # Kotlin
-systemProp.kotlin_version=1.7.10
+systemProp.kotlin_version=1.8.21
 group=org.jetbrains.kotlinx
 
 # Dependencies
-systemProp.nexus_version=1.1.0
-systemProp.dokka_version=1.7.10
-openblas_version=0.3.19
-common_csv_version=1.9.0
+systemProp.nexus_version=1.3.0
+systemProp.dokka_version=1.8.10
+openblas_version=0.3.23
+common_csv_version=1.10.0
 
 # Gradle
 org.gradle.parallel=true
@@ -22,7 +22,6 @@ kotlin.mpp.enableCInteropCommonization=true
 # Native
 kotlin.native.cacheKind.linuxX64=static
 kotlin.native.cacheKind.iosArm64=static
-kotlin.native.binary.memoryModel=experimental
 
 # Settings
 kotlin.incremental.multiplatform=true

multik-core/build.gradle.kts

@@ -32,6 +32,17 @@ kotlin {
     iosArm64()
     iosSimulatorArm64()
     iosX64()
+    wasm {
+        browser {
+            testTask {
+                /*
+                    https://youtrack.jetbrains.com/issue/KT-56633
+                    https://youtrack.jetbrains.com/issue/KT-56159
+                 */
+                this.enabled = false // fixed in 1.9.0/1.9.20
+            }
+        }
+    }
     js(IR) {
         val timeoutMs = "1000000"
         browser{
@@ -50,12 +61,6 @@ kotlin {
         }
     }
 
-    targets.withType<org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget> {
-        binaries.all {
-            freeCompilerArgs = freeCompilerArgs + "-Xallocator=std"
-        }
-    }
-
     sourceSets {
         val commonMain by getting {
         }
@@ -68,6 +73,7 @@ kotlin {
             dependencies {
                 implementation(kotlin("reflect"))
                 implementation("org.apache.commons:commons-csv:$common_csv_version")
+                implementation("org.jetbrains.bio:npy:0.3.5")
             }
         }
         val nativeMain by creating {

multik-core/src/jvmMain/kotlin/org/jetbrains/kotlinx/multik/api/Engine.kt

@@ -1,6 +1,7 @@
 package org.jetbrains.kotlinx.multik.api
 
 import java.util.*
+import java.util.concurrent.ConcurrentHashMap
 
 /**
  * Engine Provider for JVM.
@@ -17,5 +18,5 @@ public actual fun enginesProvider(): Map<EngineType, Engine> {
         )
     }
 
-    return engineList.associateBy { it.type }
+    return ConcurrentHashMap(engineList.associateBy { it.type })
 }

multik-core/src/jvmMain/kotlin/org/jetbrains/kotlinx/multik/api/io/csv.kt

@@ -22,9 +22,9 @@ import java.util.zip.GZIPInputStream
  * @param delimiter separator between elements
  * @param charset character encoding, by default is UTF_8
  */
-public inline fun <reified T : Any, reified D : Dim2> Multik.read(
+public inline fun <reified T : Any, reified D : Dim2> Multik.readCSV(
     fileName: String, delimiter: Char = ',', charset: Charset = Charsets.UTF_8
-): NDArray<T, D> = read(fileName, DataType.ofKClass(T::class), dimensionClassOf<D>(), delimiter, charset)
+): NDArray<T, D> = readCSV(fileName, DataType.ofKClass(T::class), dimensionClassOf<D>(), delimiter, charset)
 
 /**
  * Returns an NDArray of type [T] and [D] dimension read from csv file.
@@ -35,13 +35,13 @@ public inline fun <reified T : Any, reified D : Dim2> Multik.read(
  * @param delimiter separator between elements
  * @param charset character encoding, by default is UTF_8
  */
-public fun <T, D : Dim2> Multik.read(
+public fun <T, D : Dim2> Multik.readCSV(
     fileName: String, dtype: DataType, dim: Dim2,
     delimiter: Char = ',', charset: Charset = Charsets.UTF_8
 ): NDArray<T, D> {
     val file = File(fileName)
     if (!file.exists()) throw NoSuchFileException(file)
-    return read(file, dtype, dim, delimiter, charset)
+    return readCSV(file, dtype, dim, delimiter, charset)
 }
 
 /**
@@ -69,9 +69,9 @@ public fun Multik.readRaw(
  * @param delimiter separator between elements
  * @param charset character encoding, by default is UTF_8
  */
-public inline fun <reified T : Any, reified D : Dim2> Multik.read(
+public inline fun <reified T : Any, reified D : Dim2> Multik.readCSV(
     file: File, delimiter: Char = ',', charset: Charset = Charsets.UTF_8
-): NDArray<T, D> = read(file, DataType.ofKClass(T::class), dimensionClassOf<D>(), delimiter, charset)
+): NDArray<T, D> = readCSV(file, DataType.ofKClass(T::class), dimensionClassOf<D>(), delimiter, charset)
 
 /**
  * Returns an NDArray of type [T] and [D] dimension read from csv file.
@@ -82,7 +82,7 @@ public inline fun <reified T : Any, reified D : Dim2> Multik.read(
  * @param delimiter separator between elements
  * @param charset character encoding, by default is UTF_8
  */
-public fun <T, D : Dim2> Multik.read(
+public fun <T, D : Dim2> Multik.readCSV(
     file: File, dtype: DataType, dim: Dim2,
     delimiter: Char = ',', charset: Charset = Charsets.UTF_8
 ): NDArray<T, D> =
@@ -209,7 +209,7 @@ private fun isCompressed(file: File) = listOf("gz", "zip").contains(file.extensi
  * @param file file where the data will be written, if the file does not exist, it will be created
  * @param delimiter separator between elements
  */
-public fun <T, D : Dim2> Multik.write(file: File, ndarray: NDArray<T, D>, delimiter: Char = ','): Unit =
+public fun <T, D : Dim2> Multik.writeCSV(file: File, ndarray: NDArray<T, D>, delimiter: Char = ','): Unit =
     writeCSV(FileWriter(file), ndarray, CSVFormat.Builder.create(CSVFormat.DEFAULT).setDelimiter(delimiter).build())
 
 /**
@@ -219,7 +219,7 @@ public fun <T, D : Dim2> Multik.write(file: File, ndarray: NDArray<T, D>, delimi
  * @param path file path where the data will be written
  * @param delimiter separator between elements
  */
-public fun <T, D : Dim2> Multik.write(path: String, ndarray: NDArray<T, D>, delimiter: Char = ','): Unit =
+public fun <T, D : Dim2> Multik.writeCSV(path: String, ndarray: NDArray<T, D>, delimiter: Char = ','): Unit =
     writeCSV(FileWriter(path), ndarray, CSVFormat.Builder.create(CSVFormat.DEFAULT).setDelimiter(delimiter).build())
 
 /**

multik-core/src/jvmMain/kotlin/org/jetbrains/kotlinx/multik/api/io/io.kt

@@ -0,0 +1,64 @@
+package org.jetbrains.kotlinx.multik.api.io
+
+import org.jetbrains.kotlinx.multik.api.Multik
+import org.jetbrains.kotlinx.multik.ndarray.data.*
+import java.io.File
+import java.nio.file.Path
+import kotlin.io.path.Path
+import kotlin.io.path.extension
+import kotlin.io.path.notExists
+
+@PublishedApi
+internal enum class FileFormats(val extension: String) {
+    NPY("npy"),
+    NPZ("npz"),
+    CSV("csv"),
+}
+
+public inline fun <reified T : Any, reified D : Dimension> Multik.read(fileName: String): NDArray<T, D> =
+    this.read(Path(fileName))
+
+public inline fun <reified T : Any, reified D : Dimension> Multik.read(file: File): NDArray<T, D> = this.read(file.path)
+
+public inline fun <reified T : Any, reified D : Dimension> Multik.read(path: Path): NDArray<T, D> =
+    this.read(path, DataType.ofKClass(T::class), dimensionClassOf<D>())
+
+public fun <T : Any, D : Dimension> Multik.read(path: Path, dtype: DataType, dim: D): NDArray<T, D> {
+    if (path.notExists()) throw NoSuchFileException(path.toFile())
+    return when (path.extension) {
+        FileFormats.NPY.extension -> {
+            if (dtype.isComplex()) throw Exception("NPY format only supports Number types")
+            this.readNPY(path, dtype, dim)
+        }
+
+        FileFormats.CSV.extension -> {
+            if (dim.d > 2) throw Exception("CSV format only supports 1 and 2 dimensions")
+            this.readRaw(path.toFile(), dtype, dim as Dim2) as NDArray<T, D>
+        }
+
+        else -> throw Exception("Format ${path.extension} does not support reading ndarrays. If it is `npz` format, try `mk.readNPZ`")
+    }
+}
+
+public fun Multik.write(fileName: String, ndarray: NDArray<*, *>): Unit =
+    this.write(Path(fileName), ndarray)
+
+public fun Multik.write(file: File, ndarray: NDArray<*, *>): Unit =
+    this.write(file.toPath(), ndarray)
+
+public fun Multik.write(path: Path, ndarray: NDArray<*, *>): Unit =
+    when (path.extension) {
+        FileFormats.NPY.extension -> {
+            require(ndarray.dtype != DataType.ComplexFloatDataType || ndarray.dtype != DataType.ComplexFloatDataType) {
+                "NPY format does not support complex numbers."
+            }
+            this.writeNPY(path, ndarray as NDArray<out Number, *>)
+        }
+
+        FileFormats.CSV.extension -> {
+            require(ndarray.dim.d < 2) { TODO() }
+            this.writeCSV(path.toFile(), ndarray as NDArray<*, out Dim2>)
+        }
+
+        else -> throw Exception("Unknown format `${path.extension}`. Please use one of the supported formats: `npy`, `csv`.")
+    }

multik-core/src/jvmMain/kotlin/org/jetbrains/kotlinx/multik/api/io/npy.kt

@@ -0,0 +1,102 @@
+package org.jetbrains.kotlinx.multik.api.io
+
+import org.jetbrains.bio.npy.NpyArray
+import org.jetbrains.bio.npy.NpyFile
+import org.jetbrains.bio.npy.NpzFile
+import org.jetbrains.kotlinx.multik.api.Multik
+import org.jetbrains.kotlinx.multik.ndarray.data.*
+import java.io.File
+import java.nio.file.Path
+import kotlin.io.path.Path
+import kotlin.io.path.notExists
+
+public inline fun <reified T : Number, reified D : Dimension> Multik.readNPY(fileName: String): NDArray<T, D> =
+    readNPY(Path(fileName), DataType.ofKClass(T::class), dimensionClassOf<D>())
+
+public inline fun <reified T : Number, reified D : Dimension> Multik.readNPY(file: File): NDArray<T, D> =
+    readNPY(file.toPath(), DataType.ofKClass(T::class), dimensionClassOf<D>())
+
+public inline fun <reified T : Number, reified D : Dimension> Multik.readNPY(path: Path): NDArray<T, D> {
+    if (path.notExists()) throw NoSuchFileException(path.toFile())
+    return readNPY(path, DataType.ofKClass(T::class), dimensionClassOf<D>())
+}
+
+public fun <T : Any, D : Dimension> Multik.readNPY(path: Path, dtype: DataType, dim: D): NDArray<T, D> {
+    if (dtype.isComplex()) throw Exception("NPY format only supports Number types")
+    val npyArray: NpyArray = NpyFile.read(path)
+    require(npyArray.shape.size == dim.d) { "Not match dimensions: shape of npy array = ${npyArray.shape.joinToString()}, and dimension = ${dim.d}" }
+
+    val data: MemoryView<T> = when (dtype) {
+        DataType.DoubleDataType -> MemoryViewDoubleArray(npyArray.asDoubleArray())
+        DataType.FloatDataType -> MemoryViewFloatArray(npyArray.asFloatArray())
+        DataType.IntDataType -> MemoryViewIntArray(npyArray.asIntArray())
+        DataType.LongDataType -> MemoryViewLongArray(npyArray.asLongArray())
+        DataType.ShortDataType -> MemoryViewShortArray(npyArray.asShortArray())
+        DataType.ByteDataType -> MemoryViewByteArray(npyArray.asByteArray())
+        else -> throw Exception("not supported complex arrays")
+    } as MemoryView<T>
+
+    return NDArray(data, shape = npyArray.shape, dim = dim)
+}
+
+public fun Multik.readNPZ(fileName: String): List<NDArray<out Number, out DimN>> =
+    readNPZ(Path(fileName))
+
+public fun Multik.readNPZ(file: File): List<NDArray<*, out DimN>> =
+    readNPZ(file.toPath())
+
+public fun Multik.readNPZ(path: Path): List<NDArray<out Number, out DimN>> {
+    return NpzFile.read(path).use {
+        val entries = it.introspect()
+        entries.map { entry ->
+            val npyArray = it[entry.name]
+            val data = when (entry.type.kotlin) {
+                DataType.DoubleDataType.clazz -> MemoryViewDoubleArray(npyArray.asDoubleArray())
+                DataType.FloatDataType.clazz -> MemoryViewFloatArray(npyArray.asFloatArray())
+                DataType.IntDataType.clazz -> MemoryViewIntArray(npyArray.asIntArray())
+                DataType.LongDataType.clazz -> MemoryViewLongArray(npyArray.asLongArray())
+                DataType.ShortDataType.clazz -> MemoryViewShortArray(npyArray.asShortArray())
+                DataType.ByteDataType.clazz -> MemoryViewByteArray(npyArray.asByteArray())
+                else -> TODO()
+            }
+            NDArray(data, shape = npyArray.shape, dim = dimensionOf(npyArray.shape.size))
+        }
+    }
+}
+
+public fun <T : Number, D : Dimension> Multik.writeNPY(fileName: String, ndArray: NDArray<T, D>): Unit =
+    this.writeNPY(Path(fileName), ndArray)
+
+public fun <T : Number, D : Dimension> Multik.writeNPY(file: File, ndArray: NDArray<T, D>): Unit =
+    this.writeNPY(file.toPath(), ndArray)
+
+public fun <T : Number, D : Dimension> Multik.writeNPY(path: Path, ndArray: NDArray<T, D>): Unit {
+    when (ndArray.dtype) {
+        DataType.DoubleDataType -> NpyFile.write(path, ndArray.data.getDoubleArray(), ndArray.shape)
+        DataType.FloatDataType -> NpyFile.write(path, ndArray.data.getFloatArray(), ndArray.shape)
+        DataType.IntDataType -> NpyFile.write(path, ndArray.data.getIntArray(), ndArray.shape)
+        DataType.LongDataType -> NpyFile.write(path, ndArray.data.getLongArray(), ndArray.shape)
+        DataType.ShortDataType -> NpyFile.write(path, ndArray.data.getShortArray(), ndArray.shape)
+        DataType.ByteDataType -> NpyFile.write(path, ndArray.data.getByteArray(), ndArray.shape)
+        else -> TODO()
+    }
+}
+
+public fun Multik.writeNPZ(path: Path, vararg ndArrays: NDArray<out Number, out Dimension>): Unit =
+    this.writeNPZ(path, ndArrays.asList())
+
+public fun Multik.writeNPZ(path: Path, ndArrays: List<NDArray<out Number, out Dimension>>): Unit {
+    NpzFile.write(path).use {
+        ndArrays.forEachIndexed { ind, array ->
+            when (array.dtype) {
+                DataType.DoubleDataType -> it.write("arr_$ind", array.data.getDoubleArray(), array.shape)
+                DataType.FloatDataType -> it.write("arr_$ind", array.data.getFloatArray(), array.shape)
+                DataType.IntDataType -> it.write("arr_$ind", array.data.getIntArray(), array.shape)
+                DataType.LongDataType -> it.write("arr_$ind", array.data.getLongArray(), array.shape)
+                DataType.ShortDataType -> it.write("arr_$ind", array.data.getShortArray(), array.shape)
+                DataType.ByteDataType -> it.write("arr_$ind", array.data.getByteArray(), array.shape)
+                else -> TODO()
+            }
+        }
+    }
+}

multik-core/src/jvmTest/kotlin/org/jetbrains/kotlinx/multik/io/UtilsTestsIO.kt

@@ -0,0 +1,12 @@
+package org.jetbrains.kotlinx.multik.io
+
+class UtilsTestsIO
+
+internal fun testResource(resourcePath: String): String =
+    UtilsTestsIO::class.java.classLoader.getResource(resourcePath)!!.path
+
+internal fun testCsv(csvName: String) = testResource("data/csv/$csvName.csv")
+
+internal fun testNpy(npyName: String) = testResource("data/npy/$npyName.npy")
+
+internal fun testNpz(npzName: String) = testResource("data/npy/$npzName.npz")