Add native Android app scaffold (M0): whisper.cpp engine via JNI
- android/: Gradle/Kotlin project (AGP 9.4, Compose, NDK 27.1), monorepo subdir as planned; whisper.cpp v1.9.3 vendored via pinned fetch script - core/whisper: JNI wrapper (beam size, threads, language) + WhisperEngine Kotlin API mirroring the desktop engine contract - app (M0 scope): in-app GGML model download from Hugging Face, engine load, WAV picker with resampling, on-device transcription, share sheet - build validated: assembleDebug OK, libwhisper_jni.so + ggml packaged
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
plugins {
|
||||
alias(libs.plugins.android.application)
|
||||
alias(libs.plugins.kotlin.compose)
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "com.meetrec.android"
|
||||
compileSdk = 36
|
||||
|
||||
defaultConfig {
|
||||
applicationId = "com.meetrec.android"
|
||||
minSdk = 29
|
||||
targetSdk = 36
|
||||
versionCode = 1
|
||||
versionName = "0.1.0"
|
||||
}
|
||||
|
||||
buildTypes {
|
||||
release {
|
||||
isMinifyEnabled = false
|
||||
}
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
|
||||
buildFeatures {
|
||||
compose = true
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation(project(":core:whisper"))
|
||||
implementation(libs.androidx.core.ktx)
|
||||
implementation(libs.androidx.activity.compose)
|
||||
implementation(platform(libs.androidx.compose.bom))
|
||||
implementation(libs.androidx.compose.ui)
|
||||
implementation(libs.androidx.compose.material3)
|
||||
implementation(libs.kotlinx.coroutines.android)
|
||||
debugImplementation(libs.androidx.compose.ui.tooling.preview)
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
|
||||
<!-- Only used to download Whisper GGML models from Hugging Face. -->
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
|
||||
<application
|
||||
android:label="MeetRec"
|
||||
android:theme="@android:style/Theme.Material.Light.NoActionBar"
|
||||
android:allowBackup="true">
|
||||
|
||||
<activity
|
||||
android:name=".MainActivity"
|
||||
android:exported="true">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
</application>
|
||||
</manifest>
|
||||
@@ -0,0 +1,242 @@
|
||||
package com.meetrec.android
|
||||
|
||||
import android.content.Intent
|
||||
import android.os.Bundle
|
||||
import android.provider.OpenableColumns
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxSize
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.DropdownMenuItem
|
||||
import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
import androidx.compose.material3.ExposedDropdownMenuBox
|
||||
import androidx.compose.material3.ExposedDropdownMenuDefaults
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedButton
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.meetrec.core.whisper.WhisperEngine
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* M0 screen: proves the whisper.cpp engine on device.
|
||||
* Pick a model (downloads from Hugging Face), load it, pick a WAV file,
|
||||
* transcribe, share the text. Recording arrives in M1.
|
||||
*/
|
||||
class MainActivity : ComponentActivity() {
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
setContent { MeetRecScreen() }
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalMaterial3Api::class)
|
||||
@Composable
|
||||
fun MeetRecScreen() {
|
||||
val context = LocalContext.current
|
||||
val scope = rememberCoroutineScope()
|
||||
|
||||
var model by remember { mutableStateOf("tiny") }
|
||||
var modelMenu by remember { mutableStateOf(false) }
|
||||
var language by remember { mutableStateOf("") }
|
||||
var status by remember { mutableStateOf("Select a model, download and load it.") }
|
||||
var busy by remember { mutableStateOf(false) }
|
||||
var engine by remember { mutableStateOf<WhisperEngine?>(null) }
|
||||
var wavName by remember { mutableStateOf<String?>(null) }
|
||||
var wavSamples by remember { mutableStateOf<FloatArray?>(null) }
|
||||
var segments by remember { mutableStateOf<List<WhisperEngine.Segment>>(emptyList()) }
|
||||
|
||||
val modelsDir = remember { File(context.filesDir, "models") }
|
||||
val modelFile = ModelDownloader.modelFile(model, modelsDir)
|
||||
|
||||
fun <T> runBusy(block: suspend () -> T) {
|
||||
if (busy) return
|
||||
busy = true
|
||||
scope.launch {
|
||||
try {
|
||||
block()
|
||||
} catch (e: Exception) {
|
||||
status = "Failed: ${e.message}"
|
||||
} finally {
|
||||
busy = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val pickWav = rememberLauncherForActivityResult(
|
||||
ActivityResultContracts.OpenDocument()
|
||||
) { uri ->
|
||||
if (uri == null) return@rememberLauncherForActivityResult
|
||||
runBusy {
|
||||
val name = context.contentResolver
|
||||
.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME), null, null, null)
|
||||
?.use { c -> if (c.moveToFirst()) c.getString(0) else "audio" }
|
||||
?: "audio"
|
||||
val samples = withContext(Dispatchers.IO) {
|
||||
context.contentResolver.openInputStream(uri)!!.use { WavReader.read(it) }
|
||||
}
|
||||
wavName = name
|
||||
wavSamples = samples
|
||||
segments = emptyList()
|
||||
status = "$name: ${samples.size / WavReader.TARGET_RATE} s @ 16 kHz"
|
||||
}
|
||||
}
|
||||
|
||||
Column(
|
||||
modifier = Modifier.fillMaxSize().padding(16.dp),
|
||||
verticalArrangement = Arrangement.spacedBy(12.dp),
|
||||
) {
|
||||
Text("MeetRec", style = MaterialTheme.typography.headlineSmall)
|
||||
|
||||
// model + language
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
ExposedDropdownMenuBox(
|
||||
expanded = modelMenu,
|
||||
onExpandedChange = { modelMenu = it },
|
||||
modifier = Modifier.weight(1f),
|
||||
) {
|
||||
OutlinedTextField(
|
||||
value = model,
|
||||
onValueChange = {},
|
||||
readOnly = true,
|
||||
label = { Text("Model") },
|
||||
trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(modelMenu) },
|
||||
modifier = Modifier.menuAnchor().fillMaxWidth(),
|
||||
)
|
||||
ExposedDropdownMenu(expanded = modelMenu, onDismissRequest = { modelMenu = false }) {
|
||||
ModelDownloader.MODELS.keys.forEach { name ->
|
||||
DropdownMenuItem(
|
||||
text = { Text(name) },
|
||||
onClick = { model = name; modelMenu = false },
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
OutlinedTextField(
|
||||
value = language,
|
||||
onValueChange = { language = it.trim() },
|
||||
label = { Text("Language") },
|
||||
placeholder = { Text("auto") },
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
}
|
||||
|
||||
// model actions
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
Button(
|
||||
onClick = {
|
||||
runBusy {
|
||||
ModelDownloader.download(model, modelsDir) { done, total ->
|
||||
status = if (total > 0) "Downloading $model: $done / $total bytes"
|
||||
else "Downloading $model: $done bytes"
|
||||
}
|
||||
status = "Model $model ready at ${modelFile.name}"
|
||||
}
|
||||
},
|
||||
enabled = !busy && !modelFile.isFile,
|
||||
) { Text("Download") }
|
||||
Button(
|
||||
onClick = {
|
||||
runBusy {
|
||||
engine?.close()
|
||||
withContext(Dispatchers.Default) {
|
||||
engine = WhisperEngine.load(modelFile.absolutePath)
|
||||
}
|
||||
status = "$model loaded (${WhisperEngine.DEFAULT_THREADS} threads)"
|
||||
}
|
||||
},
|
||||
enabled = !busy && modelFile.isFile,
|
||||
) { Text("Load engine") }
|
||||
}
|
||||
|
||||
// wav + transcribe actions
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
OutlinedButton(
|
||||
onClick = { pickWav.launch(arrayOf("audio/*", "application/octet-stream")) },
|
||||
enabled = !busy,
|
||||
) { Text(if (wavName == null) "Pick WAV file" else "Pick another WAV") }
|
||||
Button(
|
||||
onClick = {
|
||||
val samples = wavSamples
|
||||
val eng = engine
|
||||
if (samples != null && eng != null) {
|
||||
runBusy {
|
||||
val start = System.currentTimeMillis()
|
||||
val segs = withContext(Dispatchers.Default) {
|
||||
eng.transcribe(
|
||||
samples,
|
||||
beamSize = 5,
|
||||
language = language.ifBlank { null },
|
||||
)
|
||||
}
|
||||
segments = segs
|
||||
val secs = (System.currentTimeMillis() - start) / 1000.0
|
||||
status = "${segs.size} segments in $secs s (audio " +
|
||||
"${samples.size / WavReader.TARGET_RATE} s)"
|
||||
}
|
||||
}
|
||||
},
|
||||
enabled = !busy && engine != null && wavSamples != null,
|
||||
) { Text("Transcribe") }
|
||||
}
|
||||
|
||||
if (segments.isNotEmpty()) {
|
||||
OutlinedButton(onClick = {
|
||||
val text = segments.joinToString("\n") {
|
||||
"[${fmtMs(it.startMs)}] ${it.text}"
|
||||
}
|
||||
context.startActivity(
|
||||
Intent.createChooser(
|
||||
Intent(Intent.ACTION_SEND)
|
||||
.setType("text/plain")
|
||||
.putExtra(Intent.EXTRA_TEXT, text),
|
||||
"Share transcript",
|
||||
)
|
||||
)
|
||||
}) { Text("Share transcript") }
|
||||
}
|
||||
|
||||
Text(status, style = MaterialTheme.typography.bodySmall)
|
||||
if (busy) CircularProgressIndicator()
|
||||
|
||||
LazyColumn(modifier = Modifier.fillMaxWidth().weight(1f)) {
|
||||
items(segments) { seg ->
|
||||
Column(Modifier.fillMaxWidth().padding(vertical = 4.dp)) {
|
||||
Text(
|
||||
"${fmtMs(seg.startMs)} – ${fmtMs(seg.endMs)}",
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
)
|
||||
Text(seg.text, style = MaterialTheme.typography.bodyMedium)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun fmtMs(ms: Long): String {
|
||||
val total = ms / 1000
|
||||
return "%02d:%02d".format(total / 60, total % 60)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package com.meetrec.android
|
||||
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.BufferedOutputStream
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
|
||||
/**
|
||||
* Downloads Whisper GGML models from Hugging Face into app-private storage.
|
||||
* Same files the desktop app uses, so both sides share one model ecosystem.
|
||||
*/
|
||||
object ModelDownloader {
|
||||
|
||||
val MODELS = linkedMapOf(
|
||||
"tiny" to "ggml-tiny.bin",
|
||||
"base" to "ggml-base.bin",
|
||||
"small" to "ggml-small.bin",
|
||||
"medium" to "ggml-medium.bin",
|
||||
"large-v3" to "ggml-large-v3.bin",
|
||||
)
|
||||
|
||||
private const val BASE_URL = "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/"
|
||||
|
||||
fun modelFile(modelName: String, dir: File): File = File(dir, MODELS.getValue(modelName))
|
||||
|
||||
fun isDownloaded(modelName: String, dir: File): Boolean =
|
||||
modelFile(modelName, dir).let { it.isFile && it.length() > 0 }
|
||||
|
||||
/**
|
||||
* Downloads [modelName] into [dir] (no-op if already present).
|
||||
* [onProgress] gets (bytesDone, totalBytes-or-0). Throws on failure.
|
||||
*/
|
||||
suspend fun download(
|
||||
modelName: String,
|
||||
dir: File,
|
||||
onProgress: suspend (Long, Long) -> Unit = { _, _ -> },
|
||||
): File = withContext(Dispatchers.IO) {
|
||||
val dest = modelFile(modelName, dir)
|
||||
if (isDownloaded(modelName, dir)) return@withContext dest
|
||||
dir.mkdirs()
|
||||
val tmp = File(dir, dest.name + ".part")
|
||||
|
||||
val conn = URL(BASE_URL + MODELS.getValue(modelName))
|
||||
.openConnection() as HttpURLConnection
|
||||
conn.connectTimeout = 15_000
|
||||
conn.readTimeout = 60_000
|
||||
conn.instanceFollowRedirects = true
|
||||
conn.setRequestProperty("User-Agent", "meetrec-android")
|
||||
try {
|
||||
val total = conn.contentLengthLong
|
||||
conn.inputStream.use { ins ->
|
||||
BufferedOutputStream(FileOutputStream(tmp)).use { out ->
|
||||
val buf = ByteArray(64 * 1024)
|
||||
var done = 0L
|
||||
while (true) {
|
||||
val r = ins.read(buf)
|
||||
if (r < 0) break
|
||||
out.write(buf, 0, r)
|
||||
done += r
|
||||
onProgress(done, if (total > 0) total else 0L)
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
conn.disconnect()
|
||||
}
|
||||
if (!tmp.renameTo(dest)) error("could not finalize model download")
|
||||
dest
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package com.meetrec.android
|
||||
|
||||
import java.io.InputStream
|
||||
|
||||
/**
|
||||
* Reads a PCM WAV file into 16 kHz mono float32 — the format Whisper wants.
|
||||
*
|
||||
* Supports 16-bit PCM and 32-bit IEEE-float WAVs of any channel count and
|
||||
* sample rate (multi-channel is averaged, rates are linearly resampled).
|
||||
* Pure Kotlin so it is unit-testable on the JVM.
|
||||
*/
|
||||
object WavReader {
|
||||
|
||||
const val TARGET_RATE = 16000
|
||||
|
||||
fun read(input: InputStream): FloatArray {
|
||||
val data = input.readBytes()
|
||||
require(data.size > 44) { "file too small to be a WAV" }
|
||||
require(String(data, 0, 4) == "RIFF" && String(data, 8, 4) == "WAVE") {
|
||||
"not a RIFF/WAVE file"
|
||||
}
|
||||
|
||||
var channels = 0
|
||||
var sampleRate = 0
|
||||
var audioFormat = 0
|
||||
var bitsPerSample = 0
|
||||
var samples: FloatArray? = null
|
||||
|
||||
var pos = 12
|
||||
while (pos + 8 <= data.size) {
|
||||
val id = String(data, pos, 4)
|
||||
val size = leInt(data, pos + 4)
|
||||
val body = pos + 8
|
||||
when (id) {
|
||||
"fmt " -> {
|
||||
audioFormat = leInt(data, body)
|
||||
channels = leInt(data, body + 2)
|
||||
sampleRate = leInt(data, body + 4)
|
||||
bitsPerSample = leInt(data, body + 14)
|
||||
}
|
||||
"data" -> {
|
||||
val fmt = audioFormat
|
||||
require(fmt == 1 || fmt == 3) {
|
||||
"unsupported WAV audio format $fmt (need PCM or IEEE float)"
|
||||
}
|
||||
require(bitsPerSample == 16 || bitsPerSample == 32) {
|
||||
"unsupported bit depth $bitsPerSample (need 16 or 32)"
|
||||
}
|
||||
require(channels > 0 && sampleRate > 0) { "malformed fmt chunk" }
|
||||
val len = minOf(size, data.size - body)
|
||||
samples = decode(data, body, len, fmt, bitsPerSample, channels)
|
||||
}
|
||||
}
|
||||
pos = body + size + (size and 1) // chunks are word-aligned
|
||||
}
|
||||
|
||||
val mono = samples ?: error("no data chunk found")
|
||||
return if (sampleRate == TARGET_RATE) mono else resample(mono, sampleRate, TARGET_RATE)
|
||||
}
|
||||
|
||||
private fun decode(
|
||||
data: ByteArray, body: Int, len: Int,
|
||||
audioFormat: Int, bits: Int, channels: Int,
|
||||
): FloatArray {
|
||||
val bytesPerSample = bits / 8
|
||||
val frames = len / (bytesPerSample * channels)
|
||||
val out = FloatArray(frames)
|
||||
for (f in 0 until frames) {
|
||||
var acc = 0f
|
||||
for (c in 0 until channels) {
|
||||
val o = body + (f * channels + c) * bytesPerSample
|
||||
acc += when {
|
||||
audioFormat == 1 && bits == 16 ->
|
||||
((data[o].toInt() and 0xFF) or (data[o + 1].toInt() shl 8)) / 32768f
|
||||
audioFormat == 1 && bits == 32 ->
|
||||
leInt(data, o) / 2147483648f
|
||||
audioFormat == 3 && bits == 32 ->
|
||||
Float.fromBits(leInt(data, o))
|
||||
else -> 0f
|
||||
}
|
||||
}
|
||||
out[f] = acc / channels
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private fun resample(x: FloatArray, from: Int, to: Int): FloatArray {
|
||||
if (x.isEmpty()) return x
|
||||
val n = (x.size.toDouble() * to / from).toInt().coerceAtLeast(1)
|
||||
val out = FloatArray(n)
|
||||
val step = from.toDouble() / to
|
||||
for (i in 0 until n) {
|
||||
val t = i * step
|
||||
val i0 = t.toInt().coerceAtMost(x.size - 1)
|
||||
val i1 = (i0 + 1).coerceAtMost(x.size - 1)
|
||||
val frac = (t - i0).toFloat()
|
||||
out[i] = x[i0] * (1f - frac) + x[i1] * frac
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private fun leInt(b: ByteArray, off: Int): Int =
|
||||
(b[off].toInt() and 0xFF) or
|
||||
((b[off + 1].toInt() and 0xFF) shl 8) or
|
||||
((b[off + 2].toInt() and 0xFF) shl 16) or
|
||||
((b[off + 3].toInt() and 0xFF) shl 24)
|
||||
}
|
||||
Reference in New Issue
Block a user