feat: add offline wake-word detection with sherpa-onnx

- Integrate sherpa-onnx for local "Computer" wake-word detection
- Add foreground service (microphone type) for continued listening across apps
- Request RECORD_AUDIO, POST_NOTIFICATIONS, and foreground service permissions
- Download and verify pinned sherpa-onnx model (17 MB) on first enable
- Update LauncherViewModel with wake-word state and controller
- Add WorkManager and commons-compress dependencies for model extraction
- Update README with wake-word feature details and limitations
This commit is contained in:
jahruz67
2026-07-25 14:48:05 -07:00
parent 29fc34dbdf
commit 3e1c2e714a
13 changed files with 1117 additions and 9 deletions
@@ -1,10 +1,12 @@
package com.ambient.launcher
import android.Manifest
import android.app.role.RoleManager
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.content.pm.PackageManager
import android.content.res.Configuration
import android.os.BatteryManager
import android.os.Build
@@ -56,6 +58,7 @@ import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.Home
import androidx.compose.material.icons.filled.Image as ImageIcon
import androidx.compose.material.icons.filled.Info
import androidx.compose.material.icons.filled.Mic
import androidx.compose.material.icons.filled.Refresh
import androidx.compose.material.icons.filled.Search
import androidx.compose.material.icons.filled.Settings
@@ -72,6 +75,7 @@ import androidx.compose.material3.ExperimentalMaterial3Api
import androidx.compose.material3.FilledIconButton
import androidx.compose.material3.Icon
import androidx.compose.material3.IconButton
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.NavigationRail
import androidx.compose.material3.NavigationRailItem
@@ -116,6 +120,7 @@ import androidx.compose.ui.window.Dialog
import androidx.compose.ui.window.DialogProperties
import androidx.core.graphics.drawable.toBitmap
import androidx.core.net.toUri
import androidx.core.content.ContextCompat
import androidx.core.view.WindowCompat
import androidx.core.view.WindowInsetsCompat
import androidx.core.view.WindowInsetsControllerCompat
@@ -156,6 +161,16 @@ class MainActivity : ComponentActivity() {
}
}
override fun onStart() {
super.onStart()
viewModel.setLauncherVisible(true)
}
override fun onStop() {
viewModel.setLauncherVisible(false)
super.onStop()
}
private fun requestHomeRole() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
val roleManager = getSystemService(RoleManager::class.java)
@@ -234,7 +249,12 @@ private fun AmbientLauncherApp(
updateSettings = viewModel::updateSettings,
setLocalFolder = viewModel::setLocalFolder,
refreshWeather = viewModel::refreshWeather,
requestHomeRole = requestHomeRole
requestHomeRole = requestHomeRole,
setWakeWordEnabled = viewModel::setWakeWordEnabled,
setWakeWordSensitivity = viewModel::setWakeWordSensitivity,
pauseWakeWord = viewModel::pauseWakeWord,
resumeWakeWord = viewModel::resumeWakeWord,
testWakeWord = viewModel::testWakeWord
)
}
}
@@ -642,7 +662,7 @@ private fun AppTile(
}
}
private enum class SettingsSection { DISPLAY, WEATHER, ROUTINES, APPS, ACCESS }
private enum class SettingsSection { DISPLAY, WEATHER, ROUTINES, VOICE, APPS, ACCESS }
@Composable
private fun SettingsScreen(
@@ -651,7 +671,12 @@ private fun SettingsScreen(
updateSettings: ((LauncherSettings) -> LauncherSettings) -> Unit,
setLocalFolder: (String) -> Unit,
refreshWeather: () -> Unit,
requestHomeRole: () -> Unit
requestHomeRole: () -> Unit,
setWakeWordEnabled: (Boolean) -> Unit,
setWakeWordSensitivity: (Int) -> Unit,
pauseWakeWord: () -> Unit,
resumeWakeWord: () -> Unit,
testWakeWord: () -> Unit
) {
var section by rememberSaveable { mutableStateOf(SettingsSection.DISPLAY) }
val context = LocalContext.current
@@ -677,6 +702,7 @@ private fun SettingsScreen(
SettingsSection.DISPLAY -> ImageIcon
SettingsSection.WEATHER -> Icons.Default.Refresh
SettingsSection.ROUTINES -> Icons.Default.VolumeUp
SettingsSection.VOICE -> Icons.Default.Mic
SettingsSection.APPS -> Icons.Default.Apps
SettingsSection.ACCESS -> Icons.Default.Info
}
@@ -701,6 +727,14 @@ private fun SettingsScreen(
refreshWeather
)
SettingsSection.ROUTINES -> RoutineSettings(state.settings, updateSettings)
SettingsSection.VOICE -> VoiceSettings(
state = state,
setEnabled = setWakeWordEnabled,
setSensitivity = setWakeWordSensitivity,
pause = pauseWakeWord,
resume = resumeWakeWord,
test = testWakeWord
)
SettingsSection.APPS -> ApplicationsSettings(state, updateSettings)
SettingsSection.ACCESS -> AccessSettings(requestHomeRole)
}
@@ -898,6 +932,124 @@ private fun RoutineSettings(
}
}
@Composable
private fun VoiceSettings(
state: LauncherUiState,
setEnabled: (Boolean) -> Unit,
setSensitivity: (Int) -> Unit,
pause: () -> Unit,
resume: () -> Unit,
test: () -> Unit
) = SettingsPage("Voice assistant") {
val context = LocalContext.current
val requiredPermissions = remember {
buildList {
add(Manifest.permission.RECORD_AUDIO)
if (Build.VERSION.SDK_INT >= 33) add(Manifest.permission.POST_NOTIFICATIONS)
}.toTypedArray()
}
val permissionLauncher = rememberLauncherForActivityResult(
ActivityResultContracts.RequestMultiplePermissions()
) { results ->
if (requiredPermissions.all { results[it] == true }) setEnabled(true)
}
fun permissionsGranted(): Boolean = requiredPermissions.all {
ContextCompat.checkSelfPermission(context, it) == PackageManager.PERMISSION_GRANTED
}
SettingsCard(
"Listen for “Computer”",
"Detection stays on-device. Continued listening in other apps requires a persistent " +
"notification and Androids microphone privacy indicator."
) {
Switch(
checked = state.settings.wakeWord.enabled,
onCheckedChange = { enabled ->
if (!enabled) {
setEnabled(false)
} else if (permissionsGranted()) {
setEnabled(true)
} else {
permissionLauncher.launch(requiredPermissions)
}
}
)
Text(
when (state.wakeWord.state) {
WakeWordState.DISABLED -> "Disabled"
WakeWordState.DOWNLOADING -> "Downloading model"
WakeWordState.INSTALLING -> "Installing model"
WakeWordState.READY -> "Ready"
WakeWordState.LISTENING -> "Listening"
WakeWordState.PAUSED -> "Paused"
WakeWordState.MICROPHONE_BUSY -> "Microphone busy; retrying"
WakeWordState.ERROR -> "Error"
},
fontWeight = FontWeight.SemiBold
)
if (state.wakeWord.message.isNotBlank()) {
Text(state.wakeWord.message, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
if (
state.wakeWord.modelState == ModelInstallState.DOWNLOADING ||
state.wakeWord.modelState == ModelInstallState.INSTALLING
) {
LinearProgressIndicator(
progress = { state.wakeWord.installProgress / 100f },
modifier = Modifier.fillMaxWidth()
)
Text("${state.wakeWord.installProgress}%")
}
}
SettingsCard(
"Sensitivity",
"${state.settings.wakeWord.sensitivity}% · Higher values detect more easily but may false-trigger."
) {
Slider(
value = state.settings.wakeWord.sensitivity.toFloat(),
onValueChange = { setSensitivity(it.roundToInt()) },
valueRange = 0f..100f
)
}
SettingsCard(
"Listening controls",
"Testing releases this launchers microphone before opening the selected Android assistant."
) {
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
if (state.wakeWord.state == WakeWordState.PAUSED) {
Button(onClick = resume) { Text("Resume") }
} else {
OutlinedButton(
onClick = pause,
enabled = state.settings.wakeWord.enabled
) { Text("Pause") }
}
Button(
onClick = test,
enabled = state.settings.wakeWord.enabled &&
state.wakeWord.modelState == ModelInstallState.READY
) {
Icon(Icons.Default.Mic, null)
Spacer(Modifier.width(7.dp))
Text("Test assistant")
}
}
OutlinedButton(
onClick = {
runCatching { context.startActivity(Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)) }
}
) {
Text("Open Android voice settings")
}
}
SettingsCard(
"Privacy",
"Audio frames are processed locally and are never saved or transmitted."
) {
Text("Only the last detection time and service status are retained for diagnostics.")
}
}
@Composable
private fun ApplicationsSettings(
state: LauncherUiState,
@@ -929,10 +1081,24 @@ private fun AccessSettings(requestHomeRole: () -> Unit) = SettingsPage("System a
PermissionStatus("Internet", true, "Used for weather and the built-in online photo collection.")
PermissionStatus("Local photo folder", true, "Granted only to the folder you explicitly select.")
PermissionStatus("Location", false, "Not requested. Weather uses coordinates entered in settings.")
PermissionStatus("Microphone", false, "Not requested. No wake-word model is bundled.")
PermissionStatus(
"Microphone",
ContextCompat.checkSelfPermission(
LocalContext.current,
Manifest.permission.RECORD_AUDIO
) == PackageManager.PERMISSION_GRANTED,
"Requested only when on-device wake-word listening is enabled."
)
PermissionStatus("Display over other apps", false, "Not requested. Ultra-dim only covers this launcher.")
PermissionStatus("Usage access", false, "Not requested. Recent-app history is not collected.")
PermissionStatus("Notifications", false, "Not requested because no background voice service runs.")
PermissionStatus(
"Notifications",
Build.VERSION.SDK_INT < 33 || ContextCompat.checkSelfPermission(
LocalContext.current,
Manifest.permission.POST_NOTIFICATIONS
) == PackageManager.PERMISSION_GRANTED,
"Used for the visible listening service and assistant fallback action."
)
SettingsCard(
"Audio output",
"A normal launcher cannot force other apps to use a specific device."