mirror of
https://github.com/jahruz67/Ambient-Launcher.git
synced 2026-08-08 18:14:05 +00:00
feat: add offline wake-word detection with sherpa-onnx
- Integrate sherpa-onnx for local "Computer" wake-word detection - Add foreground service (microphone type) for continued listening across apps - Request RECORD_AUDIO, POST_NOTIFICATIONS, and foreground service permissions - Download and verify pinned sherpa-onnx model (17 MB) on first enable - Update LauncherViewModel with wake-word state and controller - Add WorkManager and commons-compress dependencies for model extraction - Update README with wake-word feature details and limitations
This commit is contained in:
@@ -47,7 +47,10 @@ dependencies {
|
||||
implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.7")
|
||||
implementation("androidx.datastore:datastore-preferences:1.1.1")
|
||||
implementation("androidx.documentfile:documentfile:1.0.1")
|
||||
implementation("androidx.work:work-runtime-ktx:2.10.0")
|
||||
implementation("io.coil-kt:coil-compose:2.7.0")
|
||||
implementation("org.apache.commons:commons-compress:1.27.1")
|
||||
implementation("com.k2fsa:sherpa-onnx:1.13.4@aar")
|
||||
|
||||
debugImplementation("androidx.compose.ui:ui-tooling")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
||||
|
||||
<queries>
|
||||
<intent>
|
||||
@@ -35,5 +39,11 @@
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<service
|
||||
android:name=".voice.WakeWordService"
|
||||
android:exported="false"
|
||||
android:foregroundServiceType="microphone"
|
||||
android:stopWithTask="false" />
|
||||
</application>
|
||||
</manifest>
|
||||
|
||||
@@ -7,6 +7,8 @@ import android.content.pm.PackageManager
|
||||
import androidx.documentfile.provider.DocumentFile
|
||||
import androidx.lifecycle.AndroidViewModel
|
||||
import androidx.lifecycle.viewModelScope
|
||||
import com.ambient.launcher.voice.WakeWordController
|
||||
import com.ambient.launcher.voice.WakeWordModel
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
@@ -21,21 +23,60 @@ data class LauncherUiState(
|
||||
val localImages: List<Any> = emptyList(),
|
||||
val weather: WeatherNow? = null,
|
||||
val weatherLoading: Boolean = false,
|
||||
val weatherError: String? = null
|
||||
val weatherError: String? = null,
|
||||
val wakeWord: WakeWordSnapshot = WakeWordSnapshot()
|
||||
)
|
||||
|
||||
class LauncherViewModel(application: Application) : AndroidViewModel(application) {
|
||||
private val store = SettingsStore(application)
|
||||
private val weatherRepository = WeatherRepository()
|
||||
private val wakeWordController = WakeWordController(application)
|
||||
private val _uiState = MutableStateFlow(LauncherUiState())
|
||||
val uiState: StateFlow<LauncherUiState> = _uiState.asStateFlow()
|
||||
private var settingsLoaded = false
|
||||
|
||||
init {
|
||||
loadApps()
|
||||
viewModelScope.launch {
|
||||
wakeWordController.status.collectLatest { status ->
|
||||
_uiState.value = _uiState.value.copy(wakeWord = status)
|
||||
val current = _uiState.value.settings
|
||||
if (
|
||||
settingsLoaded &&
|
||||
status.modelState == ModelInstallState.READY &&
|
||||
current.wakeWord.installedModelVersion != WakeWordModel.VERSION
|
||||
) {
|
||||
updateSettings {
|
||||
it.copy(
|
||||
wakeWord = it.wakeWord.copy(
|
||||
installedModelVersion = WakeWordModel.VERSION
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
viewModelScope.launch {
|
||||
store.settings.collectLatest { settings ->
|
||||
settingsLoaded = true
|
||||
val old = _uiState.value.settings
|
||||
_uiState.value = _uiState.value.copy(settings = settings)
|
||||
wakeWordController.configure(
|
||||
settings.wakeWord.enabled,
|
||||
settings.wakeWord.sensitivity
|
||||
)
|
||||
if (
|
||||
_uiState.value.wakeWord.modelState == ModelInstallState.READY &&
|
||||
settings.wakeWord.installedModelVersion != WakeWordModel.VERSION
|
||||
) {
|
||||
updateSettings {
|
||||
it.copy(
|
||||
wakeWord = it.wakeWord.copy(
|
||||
installedModelVersion = WakeWordModel.VERSION
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
if (settings.localFolderUri != old.localFolderUri) loadLocalImages(settings.localFolderUri)
|
||||
if (
|
||||
settings.useFahrenheit != old.useFahrenheit ||
|
||||
@@ -98,6 +139,25 @@ class LauncherViewModel(application: Application) : AndroidViewModel(application
|
||||
}
|
||||
}
|
||||
|
||||
fun setLauncherVisible(visible: Boolean) = wakeWordController.setVisible(visible)
|
||||
|
||||
fun setWakeWordEnabled(enabled: Boolean) {
|
||||
updateSettings { settings ->
|
||||
settings.copy(wakeWord = settings.wakeWord.copy(enabled = enabled))
|
||||
}
|
||||
if (enabled) wakeWordController.enable() else wakeWordController.disable()
|
||||
}
|
||||
|
||||
fun setWakeWordSensitivity(value: Int) {
|
||||
updateSettings { settings ->
|
||||
settings.copy(wakeWord = settings.wakeWord.copy(sensitivity = value.coerceIn(0, 100)))
|
||||
}
|
||||
}
|
||||
|
||||
fun pauseWakeWord() = wakeWordController.pause()
|
||||
fun resumeWakeWord() = wakeWordController.resume()
|
||||
fun testWakeWord() = wakeWordController.testDetection()
|
||||
|
||||
fun launch(app: LauncherApp): Boolean {
|
||||
val intent = Intent(Intent.ACTION_MAIN)
|
||||
.addCategory(Intent.CATEGORY_LAUNCHER)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package com.ambient.launcher
|
||||
|
||||
import android.Manifest
|
||||
import android.app.role.RoleManager
|
||||
import android.content.BroadcastReceiver
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.IntentFilter
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.res.Configuration
|
||||
import android.os.BatteryManager
|
||||
import android.os.Build
|
||||
@@ -56,6 +58,7 @@ import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material.icons.filled.Home
|
||||
import androidx.compose.material.icons.filled.Image as ImageIcon
|
||||
import androidx.compose.material.icons.filled.Info
|
||||
import androidx.compose.material.icons.filled.Mic
|
||||
import androidx.compose.material.icons.filled.Refresh
|
||||
import androidx.compose.material.icons.filled.Search
|
||||
import androidx.compose.material.icons.filled.Settings
|
||||
@@ -72,6 +75,7 @@ import androidx.compose.material3.ExperimentalMaterial3Api
|
||||
import androidx.compose.material3.FilledIconButton
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.IconButton
|
||||
import androidx.compose.material3.LinearProgressIndicator
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.NavigationRail
|
||||
import androidx.compose.material3.NavigationRailItem
|
||||
@@ -116,6 +120,7 @@ import androidx.compose.ui.window.Dialog
|
||||
import androidx.compose.ui.window.DialogProperties
|
||||
import androidx.core.graphics.drawable.toBitmap
|
||||
import androidx.core.net.toUri
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.core.view.WindowCompat
|
||||
import androidx.core.view.WindowInsetsCompat
|
||||
import androidx.core.view.WindowInsetsControllerCompat
|
||||
@@ -156,6 +161,16 @@ class MainActivity : ComponentActivity() {
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStart() {
|
||||
super.onStart()
|
||||
viewModel.setLauncherVisible(true)
|
||||
}
|
||||
|
||||
override fun onStop() {
|
||||
viewModel.setLauncherVisible(false)
|
||||
super.onStop()
|
||||
}
|
||||
|
||||
private fun requestHomeRole() {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
val roleManager = getSystemService(RoleManager::class.java)
|
||||
@@ -234,7 +249,12 @@ private fun AmbientLauncherApp(
|
||||
updateSettings = viewModel::updateSettings,
|
||||
setLocalFolder = viewModel::setLocalFolder,
|
||||
refreshWeather = viewModel::refreshWeather,
|
||||
requestHomeRole = requestHomeRole
|
||||
requestHomeRole = requestHomeRole,
|
||||
setWakeWordEnabled = viewModel::setWakeWordEnabled,
|
||||
setWakeWordSensitivity = viewModel::setWakeWordSensitivity,
|
||||
pauseWakeWord = viewModel::pauseWakeWord,
|
||||
resumeWakeWord = viewModel::resumeWakeWord,
|
||||
testWakeWord = viewModel::testWakeWord
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -642,7 +662,7 @@ private fun AppTile(
|
||||
}
|
||||
}
|
||||
|
||||
private enum class SettingsSection { DISPLAY, WEATHER, ROUTINES, APPS, ACCESS }
|
||||
private enum class SettingsSection { DISPLAY, WEATHER, ROUTINES, VOICE, APPS, ACCESS }
|
||||
|
||||
@Composable
|
||||
private fun SettingsScreen(
|
||||
@@ -651,7 +671,12 @@ private fun SettingsScreen(
|
||||
updateSettings: ((LauncherSettings) -> LauncherSettings) -> Unit,
|
||||
setLocalFolder: (String) -> Unit,
|
||||
refreshWeather: () -> Unit,
|
||||
requestHomeRole: () -> Unit
|
||||
requestHomeRole: () -> Unit,
|
||||
setWakeWordEnabled: (Boolean) -> Unit,
|
||||
setWakeWordSensitivity: (Int) -> Unit,
|
||||
pauseWakeWord: () -> Unit,
|
||||
resumeWakeWord: () -> Unit,
|
||||
testWakeWord: () -> Unit
|
||||
) {
|
||||
var section by rememberSaveable { mutableStateOf(SettingsSection.DISPLAY) }
|
||||
val context = LocalContext.current
|
||||
@@ -677,6 +702,7 @@ private fun SettingsScreen(
|
||||
SettingsSection.DISPLAY -> ImageIcon
|
||||
SettingsSection.WEATHER -> Icons.Default.Refresh
|
||||
SettingsSection.ROUTINES -> Icons.Default.VolumeUp
|
||||
SettingsSection.VOICE -> Icons.Default.Mic
|
||||
SettingsSection.APPS -> Icons.Default.Apps
|
||||
SettingsSection.ACCESS -> Icons.Default.Info
|
||||
}
|
||||
@@ -701,6 +727,14 @@ private fun SettingsScreen(
|
||||
refreshWeather
|
||||
)
|
||||
SettingsSection.ROUTINES -> RoutineSettings(state.settings, updateSettings)
|
||||
SettingsSection.VOICE -> VoiceSettings(
|
||||
state = state,
|
||||
setEnabled = setWakeWordEnabled,
|
||||
setSensitivity = setWakeWordSensitivity,
|
||||
pause = pauseWakeWord,
|
||||
resume = resumeWakeWord,
|
||||
test = testWakeWord
|
||||
)
|
||||
SettingsSection.APPS -> ApplicationsSettings(state, updateSettings)
|
||||
SettingsSection.ACCESS -> AccessSettings(requestHomeRole)
|
||||
}
|
||||
@@ -898,6 +932,124 @@ private fun RoutineSettings(
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun VoiceSettings(
|
||||
state: LauncherUiState,
|
||||
setEnabled: (Boolean) -> Unit,
|
||||
setSensitivity: (Int) -> Unit,
|
||||
pause: () -> Unit,
|
||||
resume: () -> Unit,
|
||||
test: () -> Unit
|
||||
) = SettingsPage("Voice assistant") {
|
||||
val context = LocalContext.current
|
||||
val requiredPermissions = remember {
|
||||
buildList {
|
||||
add(Manifest.permission.RECORD_AUDIO)
|
||||
if (Build.VERSION.SDK_INT >= 33) add(Manifest.permission.POST_NOTIFICATIONS)
|
||||
}.toTypedArray()
|
||||
}
|
||||
val permissionLauncher = rememberLauncherForActivityResult(
|
||||
ActivityResultContracts.RequestMultiplePermissions()
|
||||
) { results ->
|
||||
if (requiredPermissions.all { results[it] == true }) setEnabled(true)
|
||||
}
|
||||
fun permissionsGranted(): Boolean = requiredPermissions.all {
|
||||
ContextCompat.checkSelfPermission(context, it) == PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
|
||||
SettingsCard(
|
||||
"Listen for “Computer”",
|
||||
"Detection stays on-device. Continued listening in other apps requires a persistent " +
|
||||
"notification and Android’s microphone privacy indicator."
|
||||
) {
|
||||
Switch(
|
||||
checked = state.settings.wakeWord.enabled,
|
||||
onCheckedChange = { enabled ->
|
||||
if (!enabled) {
|
||||
setEnabled(false)
|
||||
} else if (permissionsGranted()) {
|
||||
setEnabled(true)
|
||||
} else {
|
||||
permissionLauncher.launch(requiredPermissions)
|
||||
}
|
||||
}
|
||||
)
|
||||
Text(
|
||||
when (state.wakeWord.state) {
|
||||
WakeWordState.DISABLED -> "Disabled"
|
||||
WakeWordState.DOWNLOADING -> "Downloading model"
|
||||
WakeWordState.INSTALLING -> "Installing model"
|
||||
WakeWordState.READY -> "Ready"
|
||||
WakeWordState.LISTENING -> "Listening"
|
||||
WakeWordState.PAUSED -> "Paused"
|
||||
WakeWordState.MICROPHONE_BUSY -> "Microphone busy; retrying"
|
||||
WakeWordState.ERROR -> "Error"
|
||||
},
|
||||
fontWeight = FontWeight.SemiBold
|
||||
)
|
||||
if (state.wakeWord.message.isNotBlank()) {
|
||||
Text(state.wakeWord.message, color = MaterialTheme.colorScheme.onSurfaceVariant)
|
||||
}
|
||||
if (
|
||||
state.wakeWord.modelState == ModelInstallState.DOWNLOADING ||
|
||||
state.wakeWord.modelState == ModelInstallState.INSTALLING
|
||||
) {
|
||||
LinearProgressIndicator(
|
||||
progress = { state.wakeWord.installProgress / 100f },
|
||||
modifier = Modifier.fillMaxWidth()
|
||||
)
|
||||
Text("${state.wakeWord.installProgress}%")
|
||||
}
|
||||
}
|
||||
SettingsCard(
|
||||
"Sensitivity",
|
||||
"${state.settings.wakeWord.sensitivity}% · Higher values detect more easily but may false-trigger."
|
||||
) {
|
||||
Slider(
|
||||
value = state.settings.wakeWord.sensitivity.toFloat(),
|
||||
onValueChange = { setSensitivity(it.roundToInt()) },
|
||||
valueRange = 0f..100f
|
||||
)
|
||||
}
|
||||
SettingsCard(
|
||||
"Listening controls",
|
||||
"Testing releases this launcher’s microphone before opening the selected Android assistant."
|
||||
) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
if (state.wakeWord.state == WakeWordState.PAUSED) {
|
||||
Button(onClick = resume) { Text("Resume") }
|
||||
} else {
|
||||
OutlinedButton(
|
||||
onClick = pause,
|
||||
enabled = state.settings.wakeWord.enabled
|
||||
) { Text("Pause") }
|
||||
}
|
||||
Button(
|
||||
onClick = test,
|
||||
enabled = state.settings.wakeWord.enabled &&
|
||||
state.wakeWord.modelState == ModelInstallState.READY
|
||||
) {
|
||||
Icon(Icons.Default.Mic, null)
|
||||
Spacer(Modifier.width(7.dp))
|
||||
Text("Test assistant")
|
||||
}
|
||||
}
|
||||
OutlinedButton(
|
||||
onClick = {
|
||||
runCatching { context.startActivity(Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)) }
|
||||
}
|
||||
) {
|
||||
Text("Open Android voice settings")
|
||||
}
|
||||
}
|
||||
SettingsCard(
|
||||
"Privacy",
|
||||
"Audio frames are processed locally and are never saved or transmitted."
|
||||
) {
|
||||
Text("Only the last detection time and service status are retained for diagnostics.")
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun ApplicationsSettings(
|
||||
state: LauncherUiState,
|
||||
@@ -929,10 +1081,24 @@ private fun AccessSettings(requestHomeRole: () -> Unit) = SettingsPage("System a
|
||||
PermissionStatus("Internet", true, "Used for weather and the built-in online photo collection.")
|
||||
PermissionStatus("Local photo folder", true, "Granted only to the folder you explicitly select.")
|
||||
PermissionStatus("Location", false, "Not requested. Weather uses coordinates entered in settings.")
|
||||
PermissionStatus("Microphone", false, "Not requested. No wake-word model is bundled.")
|
||||
PermissionStatus(
|
||||
"Microphone",
|
||||
ContextCompat.checkSelfPermission(
|
||||
LocalContext.current,
|
||||
Manifest.permission.RECORD_AUDIO
|
||||
) == PackageManager.PERMISSION_GRANTED,
|
||||
"Requested only when on-device wake-word listening is enabled."
|
||||
)
|
||||
PermissionStatus("Display over other apps", false, "Not requested. Ultra-dim only covers this launcher.")
|
||||
PermissionStatus("Usage access", false, "Not requested. Recent-app history is not collected.")
|
||||
PermissionStatus("Notifications", false, "Not requested because no background voice service runs.")
|
||||
PermissionStatus(
|
||||
"Notifications",
|
||||
Build.VERSION.SDK_INT < 33 || ContextCompat.checkSelfPermission(
|
||||
LocalContext.current,
|
||||
Manifest.permission.POST_NOTIFICATIONS
|
||||
) == PackageManager.PERMISSION_GRANTED,
|
||||
"Used for the visible listening service and assistant fallback action."
|
||||
)
|
||||
SettingsCard(
|
||||
"Audio output",
|
||||
"A normal launcher cannot force other apps to use a specific device."
|
||||
|
||||
@@ -36,6 +36,22 @@ data class DailyWeather(
|
||||
|
||||
enum class KeepAwakeMode { WHILE_VISIBLE, WHILE_CHARGING, SYSTEM_DEFAULT }
|
||||
enum class ClockCorner { BOTTOM_LEFT, BOTTOM_RIGHT, TOP_LEFT, TOP_RIGHT }
|
||||
enum class ModelInstallState { NOT_INSTALLED, DOWNLOADING, INSTALLING, READY, FAILED }
|
||||
enum class WakeWordState { DISABLED, DOWNLOADING, INSTALLING, READY, LISTENING, PAUSED, MICROPHONE_BUSY, ERROR }
|
||||
|
||||
data class WakeWordSettings(
|
||||
val enabled: Boolean = false,
|
||||
val sensitivity: Int = 50,
|
||||
val installedModelVersion: String = ""
|
||||
)
|
||||
|
||||
data class WakeWordSnapshot(
|
||||
val state: WakeWordState = WakeWordState.DISABLED,
|
||||
val modelState: ModelInstallState = ModelInstallState.NOT_INSTALLED,
|
||||
val installProgress: Int = 0,
|
||||
val message: String = "",
|
||||
val lastDetectionAt: Long? = null
|
||||
)
|
||||
|
||||
data class LauncherSettings(
|
||||
val intervalSeconds: Int = 60,
|
||||
@@ -53,7 +69,8 @@ data class LauncherSettings(
|
||||
val nightRoutineEnabled: Boolean = false,
|
||||
val nightStartHour: Int = 23,
|
||||
val nightEndHour: Int = 7,
|
||||
val nightDimPercent: Int = 88
|
||||
val nightDimPercent: Int = 88,
|
||||
val wakeWord: WakeWordSettings = WakeWordSettings()
|
||||
)
|
||||
|
||||
sealed interface Slide {
|
||||
|
||||
@@ -29,6 +29,9 @@ class SettingsStore(private val context: Context) {
|
||||
val nightStart = intPreferencesKey("night_start")
|
||||
val nightEnd = intPreferencesKey("night_end")
|
||||
val nightDim = intPreferencesKey("night_dim")
|
||||
val wakeEnabled = booleanPreferencesKey("wake_enabled")
|
||||
val wakeSensitivity = intPreferencesKey("wake_sensitivity")
|
||||
val wakeModelVersion = stringPreferencesKey("wake_model_version")
|
||||
}
|
||||
|
||||
val settings: Flow<LauncherSettings> = context.dataStore.data.map { p ->
|
||||
@@ -52,7 +55,12 @@ class SettingsStore(private val context: Context) {
|
||||
nightRoutineEnabled = p[Keys.nightEnabled] ?: false,
|
||||
nightStartHour = p[Keys.nightStart] ?: 23,
|
||||
nightEndHour = p[Keys.nightEnd] ?: 7,
|
||||
nightDimPercent = p[Keys.nightDim] ?: 88
|
||||
nightDimPercent = p[Keys.nightDim] ?: 88,
|
||||
wakeWord = WakeWordSettings(
|
||||
enabled = p[Keys.wakeEnabled] ?: false,
|
||||
sensitivity = p[Keys.wakeSensitivity] ?: 50,
|
||||
installedModelVersion = p[Keys.wakeModelVersion] ?: ""
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -74,9 +82,16 @@ class SettingsStore(private val context: Context) {
|
||||
p[Keys.nightStart] = value.nightStartHour
|
||||
p[Keys.nightEnd] = value.nightEndHour
|
||||
p[Keys.nightDim] = value.nightDimPercent
|
||||
p[Keys.wakeEnabled] = value.wakeWord.enabled
|
||||
p[Keys.wakeSensitivity] = value.wakeWord.sensitivity
|
||||
p[Keys.wakeModelVersion] = value.wakeWord.installedModelVersion
|
||||
}
|
||||
}
|
||||
|
||||
suspend fun setWakeWordEnabled(enabled: Boolean) {
|
||||
context.dataStore.edit { it[Keys.wakeEnabled] = enabled }
|
||||
}
|
||||
|
||||
private inline fun <reified T : Enum<T>> enumOrDefault(raw: String?, default: T): T =
|
||||
enumValues<T>().firstOrNull { it.name == raw } ?: default
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package com.ambient.launcher.voice
|
||||
|
||||
import android.app.PendingIntent
|
||||
import android.content.ActivityNotFoundException
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
|
||||
enum class AssistLaunchResult { LAUNCHED, NOT_CONFIGURED, BACKGROUND_BLOCKED }
|
||||
|
||||
class DefaultAssistantLauncher(private val context: Context) {
|
||||
fun launch(): AssistLaunchResult {
|
||||
val intent = assistantIntent()
|
||||
if (intent.resolveActivity(context.packageManager) == null) {
|
||||
return AssistLaunchResult.NOT_CONFIGURED
|
||||
}
|
||||
return try {
|
||||
context.startActivity(intent)
|
||||
AssistLaunchResult.LAUNCHED
|
||||
} catch (_: ActivityNotFoundException) {
|
||||
AssistLaunchResult.NOT_CONFIGURED
|
||||
} catch (_: SecurityException) {
|
||||
AssistLaunchResult.BACKGROUND_BLOCKED
|
||||
} catch (_: RuntimeException) {
|
||||
AssistLaunchResult.BACKGROUND_BLOCKED
|
||||
}
|
||||
}
|
||||
|
||||
fun pendingIntent(): PendingIntent = PendingIntent.getActivity(
|
||||
context,
|
||||
8031,
|
||||
assistantIntent(),
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
|
||||
private fun assistantIntent() = Intent(Intent.ACTION_ASSIST)
|
||||
.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP)
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.ambient.launcher.voice
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.os.Build
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.work.Constraints
|
||||
import androidx.work.ExistingWorkPolicy
|
||||
import androidx.work.NetworkType
|
||||
import androidx.work.OneTimeWorkRequestBuilder
|
||||
import androidx.work.WorkInfo
|
||||
import androidx.work.WorkManager
|
||||
import com.ambient.launcher.ModelInstallState
|
||||
import com.ambient.launcher.WakeWordSnapshot
|
||||
import com.ambient.launcher.WakeWordState
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
|
||||
object WakeWordRuntime {
|
||||
private val mutable = MutableStateFlow(WakeWordSnapshot())
|
||||
val snapshot: StateFlow<WakeWordSnapshot> = mutable.asStateFlow()
|
||||
|
||||
fun update(
|
||||
state: WakeWordState? = null,
|
||||
modelState: ModelInstallState? = null,
|
||||
progress: Int? = null,
|
||||
message: String? = null,
|
||||
lastDetectionAt: Long? = null
|
||||
) {
|
||||
val old = mutable.value
|
||||
mutable.value = old.copy(
|
||||
state = state ?: old.state,
|
||||
modelState = modelState ?: old.modelState,
|
||||
installProgress = progress ?: old.installProgress,
|
||||
message = message ?: old.message,
|
||||
lastDetectionAt = lastDetectionAt ?: old.lastDetectionAt
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
class WakeWordController(private val context: Context) {
|
||||
val status: StateFlow<WakeWordSnapshot> = WakeWordRuntime.snapshot
|
||||
private var enabled = false
|
||||
private var visible = false
|
||||
private var sensitivity = 50
|
||||
|
||||
init {
|
||||
val lastDetection = context.getSharedPreferences("wake_runtime", Context.MODE_PRIVATE)
|
||||
.getLong("last_detection_at", 0L)
|
||||
.takeIf { it > 0L }
|
||||
if (lastDetection != null) {
|
||||
WakeWordRuntime.update(lastDetectionAt = lastDetection)
|
||||
}
|
||||
if (WakeWordModel.isReady(context)) {
|
||||
WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.READY,
|
||||
state = WakeWordState.READY,
|
||||
progress = 100,
|
||||
message = "Offline model ready"
|
||||
)
|
||||
}
|
||||
WorkManager.getInstance(context)
|
||||
.getWorkInfosForUniqueWorkLiveData(WakeWordModel.WORK_NAME)
|
||||
.observeForever { work ->
|
||||
val latest = work.maxByOrNull { it.runAttemptCount } ?: return@observeForever
|
||||
when (latest.state) {
|
||||
WorkInfo.State.ENQUEUED, WorkInfo.State.BLOCKED -> WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.DOWNLOADING,
|
||||
state = WakeWordState.DOWNLOADING,
|
||||
message = "Waiting for a network connection"
|
||||
)
|
||||
WorkInfo.State.RUNNING -> WakeWordRuntime.update(
|
||||
progress = latest.progress.getInt(
|
||||
"progress",
|
||||
WakeWordRuntime.snapshot.value.installProgress
|
||||
)
|
||||
)
|
||||
WorkInfo.State.SUCCEEDED -> {
|
||||
WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.READY,
|
||||
state = WakeWordState.READY,
|
||||
progress = 100,
|
||||
message = "Offline model ready"
|
||||
)
|
||||
if (enabled && visible) startFromVisibleActivity()
|
||||
}
|
||||
WorkInfo.State.FAILED, WorkInfo.State.CANCELLED -> WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.FAILED,
|
||||
state = WakeWordState.ERROR,
|
||||
message = latest.outputData.getString("error") ?: "Model installation failed"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun configure(isEnabled: Boolean, newSensitivity: Int) {
|
||||
enabled = isEnabled
|
||||
sensitivity = newSensitivity.coerceIn(0, 100)
|
||||
if (!enabled) disable()
|
||||
else if (visible) enable()
|
||||
}
|
||||
|
||||
fun setVisible(isVisible: Boolean) {
|
||||
visible = isVisible
|
||||
if (visible && enabled) enable()
|
||||
}
|
||||
|
||||
fun enable() {
|
||||
enabled = true
|
||||
if (!WakeWordModel.isReady(context)) {
|
||||
enqueueModelInstall()
|
||||
return
|
||||
}
|
||||
if (visible) startFromVisibleActivity()
|
||||
}
|
||||
|
||||
fun disable() {
|
||||
enabled = false
|
||||
context.stopService(Intent(context, WakeWordService::class.java))
|
||||
WakeWordRuntime.update(state = WakeWordState.DISABLED, message = "Wake word disabled")
|
||||
}
|
||||
|
||||
fun pause() = sendAction(WakeWordService.ACTION_PAUSE)
|
||||
fun resume() = if (visible) sendAction(WakeWordService.ACTION_RESUME) else Unit
|
||||
fun testDetection() = sendAction(WakeWordService.ACTION_TEST)
|
||||
|
||||
fun startFromVisibleActivity() {
|
||||
if (!enabled || !visible || !WakeWordModel.isReady(context)) return
|
||||
if (!hasPermissions()) {
|
||||
WakeWordRuntime.update(
|
||||
state = WakeWordState.ERROR,
|
||||
message = "Microphone and notification permission are required"
|
||||
)
|
||||
return
|
||||
}
|
||||
val intent = Intent(context, WakeWordService::class.java)
|
||||
.setAction(WakeWordService.ACTION_START)
|
||||
.putExtra(WakeWordService.EXTRA_SENSITIVITY, sensitivity)
|
||||
runCatching { ContextCompat.startForegroundService(context, intent) }
|
||||
.onFailure {
|
||||
WakeWordRuntime.update(
|
||||
state = WakeWordState.ERROR,
|
||||
message = "Open the launcher to restart listening"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private fun sendAction(action: String) {
|
||||
val intent = Intent(context, WakeWordService::class.java).setAction(action)
|
||||
runCatching { ContextCompat.startForegroundService(context, intent) }
|
||||
}
|
||||
|
||||
private fun enqueueModelInstall() {
|
||||
val request = OneTimeWorkRequestBuilder<WakeWordModelWorker>()
|
||||
.setConstraints(
|
||||
Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build()
|
||||
)
|
||||
.build()
|
||||
WorkManager.getInstance(context).enqueueUniqueWork(
|
||||
WakeWordModel.WORK_NAME,
|
||||
ExistingWorkPolicy.KEEP,
|
||||
request
|
||||
)
|
||||
}
|
||||
|
||||
private fun hasPermissions(): Boolean {
|
||||
val microphone = ContextCompat.checkSelfPermission(
|
||||
context,
|
||||
Manifest.permission.RECORD_AUDIO
|
||||
) == PackageManager.PERMISSION_GRANTED
|
||||
val notifications = Build.VERSION.SDK_INT < 33 ||
|
||||
ContextCompat.checkSelfPermission(
|
||||
context,
|
||||
Manifest.permission.POST_NOTIFICATIONS
|
||||
) == PackageManager.PERMISSION_GRANTED
|
||||
return microphone && notifications
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package com.ambient.launcher.voice
|
||||
|
||||
import android.content.Context
|
||||
import androidx.work.CoroutineWorker
|
||||
import androidx.work.Data
|
||||
import androidx.work.WorkerParameters
|
||||
import com.ambient.launcher.ModelInstallState
|
||||
import com.ambient.launcher.WakeWordState
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
|
||||
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
|
||||
import java.io.BufferedInputStream
|
||||
import java.io.File
|
||||
import java.net.HttpURLConnection
|
||||
import java.net.URL
|
||||
import java.security.MessageDigest
|
||||
|
||||
object WakeWordModel {
|
||||
const val VERSION = "gigaspeech-3.3M-2024-01-01"
|
||||
const val WORK_NAME = "install-wake-word-model"
|
||||
const val DOWNLOAD_URL =
|
||||
"https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/" +
|
||||
"sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz2"
|
||||
const val ARCHIVE_SHA256 = "f170013b4716e41b62b9bfd809687c207cef798ef9bc6534d524e17af9b6561a"
|
||||
private const val ROOT = "wake-word-model"
|
||||
|
||||
val requiredFiles = setOf(
|
||||
"encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx",
|
||||
"decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx",
|
||||
"joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx",
|
||||
"tokens.txt",
|
||||
"bpe.model"
|
||||
)
|
||||
|
||||
fun directory(context: Context) = File(context.filesDir, ROOT)
|
||||
fun isReady(context: Context): Boolean {
|
||||
val dir = directory(context)
|
||||
return File(dir, ".ready").readTextOrNull() == VERSION &&
|
||||
requiredFiles.all { File(dir, it).isFile } &&
|
||||
File(dir, "keywords.txt").isFile
|
||||
}
|
||||
|
||||
fun path(context: Context, name: String) = File(directory(context), name).absolutePath
|
||||
|
||||
private fun File.readTextOrNull(): String? = runCatching { readText() }.getOrNull()
|
||||
}
|
||||
|
||||
class WakeWordModelWorker(
|
||||
appContext: Context,
|
||||
params: WorkerParameters
|
||||
) : CoroutineWorker(appContext, params) {
|
||||
override suspend fun doWork(): Result = withContext(Dispatchers.IO) {
|
||||
val parent = applicationContext.filesDir
|
||||
val archive = File(parent, "wake-word-model.download")
|
||||
val staging = File(parent, "wake-word-model.staging")
|
||||
try {
|
||||
WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.DOWNLOADING,
|
||||
state = WakeWordState.DOWNLOADING,
|
||||
message = "Downloading offline wake-word model"
|
||||
)
|
||||
download(archive)
|
||||
if (sha256(archive) != WakeWordModel.ARCHIVE_SHA256) {
|
||||
error("Downloaded model failed integrity verification")
|
||||
}
|
||||
WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.INSTALLING,
|
||||
state = WakeWordState.INSTALLING,
|
||||
message = "Installing offline wake-word model"
|
||||
)
|
||||
staging.deleteRecursively()
|
||||
staging.mkdirs()
|
||||
extractSelected(archive, staging)
|
||||
createComputerKeyword(File(staging, "tokens.txt"), File(staging, "keywords.txt"))
|
||||
if (!WakeWordModel.requiredFiles.all { File(staging, it).isFile }) {
|
||||
error("Model archive is incomplete")
|
||||
}
|
||||
File(staging, ".ready").writeText(WakeWordModel.VERSION)
|
||||
val destination = WakeWordModel.directory(applicationContext)
|
||||
destination.deleteRecursively()
|
||||
if (!staging.renameTo(destination)) error("Unable to activate installed model")
|
||||
archive.delete()
|
||||
WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.READY,
|
||||
state = WakeWordState.READY,
|
||||
progress = 100,
|
||||
message = "Offline model ready"
|
||||
)
|
||||
Result.success(Data.Builder().putString("version", WakeWordModel.VERSION).build())
|
||||
} catch (error: Throwable) {
|
||||
archive.delete()
|
||||
staging.deleteRecursively()
|
||||
WakeWordRuntime.update(
|
||||
modelState = ModelInstallState.FAILED,
|
||||
state = WakeWordState.ERROR,
|
||||
message = error.message ?: "Model installation failed"
|
||||
)
|
||||
Result.failure(Data.Builder().putString("error", error.message).build())
|
||||
}
|
||||
}
|
||||
|
||||
private fun download(target: File) {
|
||||
val connection = URL(WakeWordModel.DOWNLOAD_URL).openConnection() as HttpURLConnection
|
||||
connection.connectTimeout = 15_000
|
||||
connection.readTimeout = 30_000
|
||||
connection.instanceFollowRedirects = true
|
||||
connection.setRequestProperty("Accept", "application/octet-stream")
|
||||
try {
|
||||
if (connection.responseCode !in 200..299) {
|
||||
error("Model download returned HTTP ${connection.responseCode}")
|
||||
}
|
||||
val total = connection.contentLengthLong.coerceAtLeast(1L)
|
||||
var received = 0L
|
||||
var lastProgress = -1
|
||||
connection.inputStream.buffered().use { input ->
|
||||
target.outputStream().buffered().use { output ->
|
||||
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
|
||||
while (true) {
|
||||
if (isStopped) error("Model download cancelled")
|
||||
val count = input.read(buffer)
|
||||
if (count < 0) break
|
||||
output.write(buffer, 0, count)
|
||||
received += count
|
||||
val progress = ((received * 100L) / total).toInt().coerceIn(0, 99)
|
||||
if (progress != lastProgress) {
|
||||
lastProgress = progress
|
||||
setProgressAsync(Data.Builder().putInt("progress", progress).build())
|
||||
WakeWordRuntime.update(progress = progress)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
connection.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
private fun extractSelected(archive: File, destination: File) {
|
||||
TarArchiveInputStream(
|
||||
BZip2CompressorInputStream(BufferedInputStream(archive.inputStream()))
|
||||
).use { tar ->
|
||||
while (true) {
|
||||
val entry = tar.nextTarEntry ?: break
|
||||
if (!entry.isFile) continue
|
||||
val name = entry.name.substringAfterLast('/')
|
||||
if (name !in WakeWordModel.requiredFiles) continue
|
||||
val output = File(destination, name)
|
||||
output.outputStream().buffered().use { tar.copyTo(it) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun createComputerKeyword(tokensFile: File, output: File) {
|
||||
val tokens = tokensFile.readLines().mapIndexedNotNull { index, line ->
|
||||
val columns = line.trim().split(Regex("\\s+"))
|
||||
val piece = columns.firstOrNull()?.takeIf { it.isNotBlank() } ?: return@mapIndexedNotNull null
|
||||
piece to (columns.getOrNull(1)?.toIntOrNull() ?: index)
|
||||
}
|
||||
val target = "▁COMPUTER"
|
||||
data class Candidate(val pieces: List<Pair<String, Int>>, val score: Int)
|
||||
val best = arrayOfNulls<Candidate>(target.length + 1)
|
||||
best[0] = Candidate(emptyList(), 0)
|
||||
for (start in target.indices) {
|
||||
val current = best[start] ?: continue
|
||||
tokens.forEach { token ->
|
||||
val piece = token.first
|
||||
if (target.startsWith(piece, start)) {
|
||||
val end = start + piece.length
|
||||
val candidate = Candidate(current.pieces + token, current.score + token.second)
|
||||
val previous = best[end]
|
||||
if (
|
||||
previous == null ||
|
||||
candidate.pieces.size < previous.pieces.size ||
|
||||
candidate.pieces.size == previous.pieces.size && candidate.score < previous.score
|
||||
) {
|
||||
best[end] = candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val pieces = best[target.length]?.pieces ?: error("Model cannot tokenize COMPUTER")
|
||||
output.writeText(pieces.joinToString(" ") { it.first } + "\n")
|
||||
}
|
||||
|
||||
private fun sha256(file: File): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256")
|
||||
file.inputStream().buffered().use { input ->
|
||||
val buffer = ByteArray(DEFAULT_BUFFER_SIZE)
|
||||
while (true) {
|
||||
val count = input.read(buffer)
|
||||
if (count < 0) break
|
||||
digest.update(buffer, 0, count)
|
||||
}
|
||||
}
|
||||
return digest.digest().joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package com.ambient.launcher.voice
|
||||
|
||||
import android.Manifest
|
||||
import android.app.Notification
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.app.PendingIntent
|
||||
import android.app.Service
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.pm.ServiceInfo
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioRecord
|
||||
import android.media.MediaRecorder
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.IBinder
|
||||
import android.os.Looper
|
||||
import androidx.core.app.NotificationCompat
|
||||
import androidx.core.app.ServiceCompat
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.ambient.launcher.MainActivity
|
||||
import com.ambient.launcher.R
|
||||
import com.ambient.launcher.SettingsStore
|
||||
import com.ambient.launcher.WakeWordState
|
||||
import com.k2fsa.sherpa.onnx.FeatureConfig
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotter
|
||||
import com.k2fsa.sherpa.onnx.KeywordSpotterConfig
|
||||
import com.k2fsa.sherpa.onnx.OnlineModelConfig
|
||||
import com.k2fsa.sherpa.onnx.OnlineStream
|
||||
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlin.concurrent.thread
|
||||
|
||||
class WakeWordService : Service() {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
private var recorder: AudioRecord? = null
|
||||
private var spotter: KeywordSpotter? = null
|
||||
private var stream: OnlineStream? = null
|
||||
private var recordingThread: Thread? = null
|
||||
@Volatile private var listening = false
|
||||
@Volatile private var paused = false
|
||||
private var sensitivity = 50
|
||||
private var lastDetection = 0L
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
createChannels()
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
promoteToForeground()
|
||||
when (intent?.action) {
|
||||
ACTION_START -> {
|
||||
val requestedSensitivity =
|
||||
intent.getIntExtra(EXTRA_SENSITIVITY, 50).coerceIn(0, 100)
|
||||
if (requestedSensitivity != sensitivity && listening) stopListening()
|
||||
sensitivity = requestedSensitivity
|
||||
paused = false
|
||||
startListening()
|
||||
}
|
||||
ACTION_PAUSE -> pauseListening()
|
||||
ACTION_RESUME -> {
|
||||
paused = false
|
||||
startListening()
|
||||
}
|
||||
ACTION_STOP -> {
|
||||
stopListening()
|
||||
stopForeground(STOP_FOREGROUND_REMOVE)
|
||||
serviceScope.launch {
|
||||
SettingsStore(applicationContext).setWakeWordEnabled(false)
|
||||
stopSelf()
|
||||
}
|
||||
}
|
||||
ACTION_TEST -> handleDetection(force = true)
|
||||
else -> {
|
||||
WakeWordRuntime.update(
|
||||
state = WakeWordState.ERROR,
|
||||
message = "Listening must be started from the visible launcher"
|
||||
)
|
||||
stopSelf()
|
||||
}
|
||||
}
|
||||
return START_STICKY
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
stopListening()
|
||||
mainHandler.removeCallbacksAndMessages(null)
|
||||
serviceScope.cancel()
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
|
||||
private fun startListening() {
|
||||
if (listening || paused) return
|
||||
if (!WakeWordModel.isReady(this)) {
|
||||
WakeWordRuntime.update(state = WakeWordState.ERROR, message = "Wake-word model is not ready")
|
||||
return
|
||||
}
|
||||
if (
|
||||
ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) !=
|
||||
PackageManager.PERMISSION_GRANTED
|
||||
) {
|
||||
WakeWordRuntime.update(state = WakeWordState.ERROR, message = "Microphone permission is required")
|
||||
return
|
||||
}
|
||||
runCatching {
|
||||
val minimum = AudioRecord.getMinBufferSize(
|
||||
SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT
|
||||
)
|
||||
if (minimum <= 0) error("Microphone is unavailable")
|
||||
val audioRecord = AudioRecord(
|
||||
MediaRecorder.AudioSource.VOICE_RECOGNITION,
|
||||
SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT,
|
||||
minimum * 2
|
||||
)
|
||||
if (audioRecord.state != AudioRecord.STATE_INITIALIZED) {
|
||||
audioRecord.release()
|
||||
error("Microphone is busy")
|
||||
}
|
||||
val model = createSpotter()
|
||||
val onlineStream = model.createStream()
|
||||
recorder = audioRecord
|
||||
spotter = model
|
||||
stream = onlineStream
|
||||
audioRecord.startRecording()
|
||||
if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) {
|
||||
error("Microphone is busy")
|
||||
}
|
||||
listening = true
|
||||
WakeWordRuntime.update(
|
||||
state = WakeWordState.LISTENING,
|
||||
message = "Listening for “Computer”"
|
||||
)
|
||||
updateNotification()
|
||||
recordingThread = thread(start = true, name = "WakeWordAudio") {
|
||||
processAudio()
|
||||
}
|
||||
}.onFailure {
|
||||
releaseAudio()
|
||||
WakeWordRuntime.update(
|
||||
state = WakeWordState.MICROPHONE_BUSY,
|
||||
message = it.message ?: "Microphone is busy"
|
||||
)
|
||||
if (!paused) mainHandler.postDelayed({ startListening() }, RETRY_DELAY_MS)
|
||||
}
|
||||
}
|
||||
|
||||
private fun createSpotter(): KeywordSpotter {
|
||||
val threshold = (0.45f - sensitivity * 0.003f).coerceIn(0.12f, 0.45f)
|
||||
return KeywordSpotter(
|
||||
config = KeywordSpotterConfig(
|
||||
featConfig = FeatureConfig(sampleRate = SAMPLE_RATE, featureDim = 80, dither = 0f),
|
||||
modelConfig = OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = WakeWordModel.path(
|
||||
this,
|
||||
"encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"
|
||||
),
|
||||
decoder = WakeWordModel.path(
|
||||
this,
|
||||
"decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"
|
||||
),
|
||||
joiner = WakeWordModel.path(
|
||||
this,
|
||||
"joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx"
|
||||
)
|
||||
),
|
||||
tokens = WakeWordModel.path(this, "tokens.txt"),
|
||||
numThreads = 1,
|
||||
provider = "cpu",
|
||||
modelType = "zipformer2"
|
||||
),
|
||||
keywordsFile = WakeWordModel.path(this, "keywords.txt"),
|
||||
keywordsScore = 1.5f,
|
||||
keywordsThreshold = threshold,
|
||||
maxActivePaths = 4,
|
||||
numTrailingBlanks = 2
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private fun processAudio() {
|
||||
val buffer = ShortArray(1600)
|
||||
while (listening) {
|
||||
val count = recorder?.read(buffer, 0, buffer.size) ?: break
|
||||
if (count <= 0) continue
|
||||
val samples = FloatArray(count) { buffer[it] / 32768f }
|
||||
val activeStream = stream ?: break
|
||||
val activeSpotter = spotter ?: break
|
||||
activeStream.acceptWaveform(samples, SAMPLE_RATE)
|
||||
while (activeSpotter.isReady(activeStream)) {
|
||||
activeSpotter.decode(activeStream)
|
||||
if (activeSpotter.getResult(activeStream).keyword.isNotBlank()) {
|
||||
activeSpotter.reset(activeStream)
|
||||
handleDetection(force = false)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun handleDetection(force: Boolean) {
|
||||
val now = System.currentTimeMillis()
|
||||
if (!force && now - lastDetection < DETECTION_DEBOUNCE_MS) return
|
||||
lastDetection = now
|
||||
getSharedPreferences("wake_runtime", MODE_PRIVATE)
|
||||
.edit()
|
||||
.putLong("last_detection_at", now)
|
||||
.apply()
|
||||
stopListening()
|
||||
WakeWordRuntime.update(
|
||||
state = WakeWordState.READY,
|
||||
message = "Wake word detected",
|
||||
lastDetectionAt = now
|
||||
)
|
||||
val launcher = DefaultAssistantLauncher(this)
|
||||
val result = launcher.launch()
|
||||
if (result != AssistLaunchResult.LAUNCHED) showAssistantFallback(launcher)
|
||||
if (!paused) mainHandler.postDelayed({ startListening() }, ASSISTANT_COOLDOWN_MS)
|
||||
}
|
||||
|
||||
private fun pauseListening() {
|
||||
paused = true
|
||||
stopListening()
|
||||
WakeWordRuntime.update(state = WakeWordState.PAUSED, message = "Wake-word listening paused")
|
||||
updateNotification()
|
||||
}
|
||||
|
||||
private fun stopListening() {
|
||||
listening = false
|
||||
runCatching { recorder?.stop() }
|
||||
releaseAudio()
|
||||
}
|
||||
|
||||
private fun releaseAudio() {
|
||||
runCatching { recorder?.release() }
|
||||
recorder = null
|
||||
stream?.release()
|
||||
stream = null
|
||||
spotter?.release()
|
||||
spotter = null
|
||||
recordingThread = null
|
||||
}
|
||||
|
||||
private fun promoteToForeground() {
|
||||
val type = if (Build.VERSION.SDK_INT >= 30) {
|
||||
ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
} else 0
|
||||
ServiceCompat.startForeground(
|
||||
this,
|
||||
LISTENING_NOTIFICATION_ID,
|
||||
listeningNotification(),
|
||||
type
|
||||
)
|
||||
}
|
||||
|
||||
private fun updateNotification() {
|
||||
getSystemService(NotificationManager::class.java)
|
||||
.notify(LISTENING_NOTIFICATION_ID, listeningNotification())
|
||||
}
|
||||
|
||||
private fun listeningNotification(): Notification {
|
||||
val openLauncher = PendingIntent.getActivity(
|
||||
this,
|
||||
8010,
|
||||
Intent(this, MainActivity::class.java),
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
val toggleAction = if (paused) ACTION_RESUME else ACTION_PAUSE
|
||||
val toggleLabel = if (paused) "Resume" else "Pause"
|
||||
return NotificationCompat.Builder(this, LISTENING_CHANNEL)
|
||||
.setSmallIcon(R.drawable.ic_launcher)
|
||||
.setContentTitle(if (paused) "Wake word paused" else "Listening for “Computer”")
|
||||
.setContentText("Tap to return to Ambient Launcher")
|
||||
.setContentIntent(openLauncher)
|
||||
.setOngoing(true)
|
||||
.setOnlyAlertOnce(true)
|
||||
.setCategory(NotificationCompat.CATEGORY_SERVICE)
|
||||
.addAction(0, toggleLabel, servicePendingIntent(toggleAction, 8011))
|
||||
.addAction(0, "Stop", servicePendingIntent(ACTION_STOP, 8012))
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun showAssistantFallback(launcher: DefaultAssistantLauncher) {
|
||||
val notification = NotificationCompat.Builder(this, ASSISTANT_CHANNEL)
|
||||
.setSmallIcon(R.drawable.ic_launcher)
|
||||
.setContentTitle("Wake word detected")
|
||||
.setContentText("Tap to open your default assistant")
|
||||
.setContentIntent(launcher.pendingIntent())
|
||||
.setAutoCancel(true)
|
||||
.setPriority(NotificationCompat.PRIORITY_HIGH)
|
||||
.build()
|
||||
getSystemService(NotificationManager::class.java)
|
||||
.notify(ASSISTANT_NOTIFICATION_ID, notification)
|
||||
}
|
||||
|
||||
private fun servicePendingIntent(action: String, requestCode: Int): PendingIntent =
|
||||
PendingIntent.getService(
|
||||
this,
|
||||
requestCode,
|
||||
Intent(this, WakeWordService::class.java).setAction(action),
|
||||
PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
|
||||
)
|
||||
|
||||
private fun createChannels() {
|
||||
val manager = getSystemService(NotificationManager::class.java)
|
||||
manager.createNotificationChannel(
|
||||
NotificationChannel(
|
||||
LISTENING_CHANNEL,
|
||||
"Wake-word listening",
|
||||
NotificationManager.IMPORTANCE_LOW
|
||||
).apply {
|
||||
description = "Shown while Ambient Launcher listens for the local wake word"
|
||||
setSound(null, null)
|
||||
}
|
||||
)
|
||||
manager.createNotificationChannel(
|
||||
NotificationChannel(
|
||||
ASSISTANT_CHANNEL,
|
||||
"Assistant actions",
|
||||
NotificationManager.IMPORTANCE_HIGH
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val ACTION_START = "com.ambient.launcher.voice.START"
|
||||
const val ACTION_PAUSE = "com.ambient.launcher.voice.PAUSE"
|
||||
const val ACTION_RESUME = "com.ambient.launcher.voice.RESUME"
|
||||
const val ACTION_STOP = "com.ambient.launcher.voice.STOP"
|
||||
const val ACTION_TEST = "com.ambient.launcher.voice.TEST"
|
||||
const val EXTRA_SENSITIVITY = "sensitivity"
|
||||
private const val SAMPLE_RATE = 16_000
|
||||
private const val DETECTION_DEBOUNCE_MS = 5_000L
|
||||
private const val ASSISTANT_COOLDOWN_MS = 12_000L
|
||||
private const val RETRY_DELAY_MS = 4_000L
|
||||
private const val LISTENING_CHANNEL = "wake_word_listening"
|
||||
private const val ASSISTANT_CHANNEL = "wake_word_assistant"
|
||||
private const val LISTENING_NOTIFICATION_ID = 801
|
||||
private const val ASSISTANT_NOTIFICATION_ID = 802
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user