diff --git a/.github/workflows/android-apk.yml b/.github/workflows/android-apk.yml new file mode 100644 index 0000000..ced106f --- /dev/null +++ b/.github/workflows/android-apk.yml @@ -0,0 +1,51 @@ +name: Build Android APK + +on: + workflow_dispatch: + push: + branches: + - main + - master + pull_request: + +permissions: + contents: read + +concurrency: + group: android-apk-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-debug-apk: + name: Build debug APK + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up Java 17 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + + # This repository does not currently contain Gradle wrapper files. + # AGP 8.7.x requires Gradle 8.9. + - name: Set up Gradle 8.9 + uses: gradle/actions/setup-gradle@v6 + with: + gradle-version: "8.9" + cache-provider: basic + + - name: Build installable debug APK + run: gradle :app:assembleDebug --no-daemon --stacktrace + + - name: Upload APK + uses: actions/upload-artifact@v6 + with: + name: ambient-launcher-debug-${{ github.run_number }} + path: app/build/outputs/apk/debug/*.apk + if-no-files-found: error + retention-days: 14 diff --git a/README.md b/README.md index 3867f84..4d4b562 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,8 @@ An Android tablet home-screen replacement written in Kotlin and Jetpack Compose. - Floating searchable app tray with persistent pinned apps - Keep-awake options, including charging-only - Scheduled ultra-dim/night presentation while the launcher is visible +- Offline “Computer” wake-word detection that opens Android’s default assistant +- Continued listening across other apps through a visible microphone foreground service - Honest capability and permission status screen - Manual Android media output panel shortcut @@ -22,4 +24,10 @@ No API key is required. The initial location is San Francisco; change latitude, ## Deliberately not claimed -Always-listening wake-word detection, system-wide audio routing, and overlays over unrelated apps are not included. Those features require a wake-word engine/model and/or sensitive Android services and permissions. The permissions screen explains these boundaries. +System-wide audio routing and overlays over unrelated apps are not included. Android may block an assistant activity from opening over another app; when that happens, the wake-word service posts a one-tap **Open assistant** notification. + +## Wake word + +Enabling wake word downloads the pinned open-source sherpa-onnx English keyword model (about 17 MB), verifies its SHA-256 digest, and installs only the required int8 model files in private app storage. Audio is analyzed locally and is never saved or transmitted. + +The pinned sherpa-onnx 1.13.4 Android AAR is resolved from the project’s versioned GitHub release repository during a future Gradle build. Listening can only be started while the launcher is visible and requires microphone and notification permission. It does not automatically start from a boot receiver. diff --git a/app/build.gradle.kts b/app/build.gradle.kts index f841300..7845227 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -47,7 +47,10 @@ dependencies { implementation("androidx.lifecycle:lifecycle-viewmodel-compose:2.8.7") implementation("androidx.datastore:datastore-preferences:1.1.1") implementation("androidx.documentfile:documentfile:1.0.1") + implementation("androidx.work:work-runtime-ktx:2.10.0") implementation("io.coil-kt:coil-compose:2.7.0") + implementation("org.apache.commons:commons-compress:1.27.1") + implementation("com.k2fsa:sherpa-onnx:1.13.4@aar") debugImplementation("androidx.compose.ui:ui-tooling") } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 85eebf6..1b42ddc 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -3,6 +3,10 @@ + + + + @@ -35,5 +39,11 @@ + + diff --git a/app/src/main/java/com/ambient/launcher/LauncherViewModel.kt b/app/src/main/java/com/ambient/launcher/LauncherViewModel.kt index 7133df1..031ee1b 100644 --- a/app/src/main/java/com/ambient/launcher/LauncherViewModel.kt +++ b/app/src/main/java/com/ambient/launcher/LauncherViewModel.kt @@ -7,6 +7,8 @@ import android.content.pm.PackageManager import androidx.documentfile.provider.DocumentFile import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope +import com.ambient.launcher.voice.WakeWordController +import com.ambient.launcher.voice.WakeWordModel import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow @@ -21,21 +23,60 @@ data class LauncherUiState( val localImages: List = emptyList(), val weather: WeatherNow? = null, val weatherLoading: Boolean = false, - val weatherError: String? = null + val weatherError: String? = null, + val wakeWord: WakeWordSnapshot = WakeWordSnapshot() ) class LauncherViewModel(application: Application) : AndroidViewModel(application) { private val store = SettingsStore(application) private val weatherRepository = WeatherRepository() + private val wakeWordController = WakeWordController(application) private val _uiState = MutableStateFlow(LauncherUiState()) val uiState: StateFlow = _uiState.asStateFlow() + private var settingsLoaded = false init { loadApps() + viewModelScope.launch { + wakeWordController.status.collectLatest { status -> + _uiState.value = _uiState.value.copy(wakeWord = status) + val current = _uiState.value.settings + if ( + settingsLoaded && + status.modelState == ModelInstallState.READY && + current.wakeWord.installedModelVersion != WakeWordModel.VERSION + ) { + updateSettings { + it.copy( + wakeWord = it.wakeWord.copy( + installedModelVersion = WakeWordModel.VERSION + ) + ) + } + } + } + } viewModelScope.launch { store.settings.collectLatest { settings -> + settingsLoaded = true val old = _uiState.value.settings _uiState.value = _uiState.value.copy(settings = settings) + wakeWordController.configure( + settings.wakeWord.enabled, + settings.wakeWord.sensitivity + ) + if ( + _uiState.value.wakeWord.modelState == ModelInstallState.READY && + settings.wakeWord.installedModelVersion != WakeWordModel.VERSION + ) { + updateSettings { + it.copy( + wakeWord = it.wakeWord.copy( + installedModelVersion = WakeWordModel.VERSION + ) + ) + } + } if (settings.localFolderUri != old.localFolderUri) loadLocalImages(settings.localFolderUri) if ( settings.useFahrenheit != old.useFahrenheit || @@ -98,6 +139,25 @@ class LauncherViewModel(application: Application) : AndroidViewModel(application } } + fun setLauncherVisible(visible: Boolean) = wakeWordController.setVisible(visible) + + fun setWakeWordEnabled(enabled: Boolean) { + updateSettings { settings -> + settings.copy(wakeWord = settings.wakeWord.copy(enabled = enabled)) + } + if (enabled) wakeWordController.enable() else wakeWordController.disable() + } + + fun setWakeWordSensitivity(value: Int) { + updateSettings { settings -> + settings.copy(wakeWord = settings.wakeWord.copy(sensitivity = value.coerceIn(0, 100))) + } + } + + fun pauseWakeWord() = wakeWordController.pause() + fun resumeWakeWord() = wakeWordController.resume() + fun testWakeWord() = wakeWordController.testDetection() + fun launch(app: LauncherApp): Boolean { val intent = Intent(Intent.ACTION_MAIN) .addCategory(Intent.CATEGORY_LAUNCHER) diff --git a/app/src/main/java/com/ambient/launcher/MainActivity.kt b/app/src/main/java/com/ambient/launcher/MainActivity.kt index 038a4a0..608935a 100644 --- a/app/src/main/java/com/ambient/launcher/MainActivity.kt +++ b/app/src/main/java/com/ambient/launcher/MainActivity.kt @@ -1,10 +1,12 @@ package com.ambient.launcher +import android.Manifest import android.app.role.RoleManager import android.content.BroadcastReceiver import android.content.Context import android.content.Intent import android.content.IntentFilter +import android.content.pm.PackageManager import android.content.res.Configuration import android.os.BatteryManager import android.os.Build @@ -56,6 +58,7 @@ import androidx.compose.material.icons.filled.Close import androidx.compose.material.icons.filled.Home import androidx.compose.material.icons.filled.Image as ImageIcon import androidx.compose.material.icons.filled.Info +import androidx.compose.material.icons.filled.Mic import androidx.compose.material.icons.filled.Refresh import androidx.compose.material.icons.filled.Search import androidx.compose.material.icons.filled.Settings @@ -72,6 +75,7 @@ import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.FilledIconButton import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme import androidx.compose.material3.NavigationRail import androidx.compose.material3.NavigationRailItem @@ -116,6 +120,7 @@ import androidx.compose.ui.window.Dialog import androidx.compose.ui.window.DialogProperties import androidx.core.graphics.drawable.toBitmap import androidx.core.net.toUri +import androidx.core.content.ContextCompat import androidx.core.view.WindowCompat import androidx.core.view.WindowInsetsCompat import androidx.core.view.WindowInsetsControllerCompat @@ -156,6 +161,16 @@ class MainActivity : ComponentActivity() { } } + override fun onStart() { + super.onStart() + viewModel.setLauncherVisible(true) + } + + override fun onStop() { + viewModel.setLauncherVisible(false) + super.onStop() + } + private fun requestHomeRole() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { val roleManager = getSystemService(RoleManager::class.java) @@ -234,7 +249,12 @@ private fun AmbientLauncherApp( updateSettings = viewModel::updateSettings, setLocalFolder = viewModel::setLocalFolder, refreshWeather = viewModel::refreshWeather, - requestHomeRole = requestHomeRole + requestHomeRole = requestHomeRole, + setWakeWordEnabled = viewModel::setWakeWordEnabled, + setWakeWordSensitivity = viewModel::setWakeWordSensitivity, + pauseWakeWord = viewModel::pauseWakeWord, + resumeWakeWord = viewModel::resumeWakeWord, + testWakeWord = viewModel::testWakeWord ) } } @@ -642,7 +662,7 @@ private fun AppTile( } } -private enum class SettingsSection { DISPLAY, WEATHER, ROUTINES, APPS, ACCESS } +private enum class SettingsSection { DISPLAY, WEATHER, ROUTINES, VOICE, APPS, ACCESS } @Composable private fun SettingsScreen( @@ -651,7 +671,12 @@ private fun SettingsScreen( updateSettings: ((LauncherSettings) -> LauncherSettings) -> Unit, setLocalFolder: (String) -> Unit, refreshWeather: () -> Unit, - requestHomeRole: () -> Unit + requestHomeRole: () -> Unit, + setWakeWordEnabled: (Boolean) -> Unit, + setWakeWordSensitivity: (Int) -> Unit, + pauseWakeWord: () -> Unit, + resumeWakeWord: () -> Unit, + testWakeWord: () -> Unit ) { var section by rememberSaveable { mutableStateOf(SettingsSection.DISPLAY) } val context = LocalContext.current @@ -677,6 +702,7 @@ private fun SettingsScreen( SettingsSection.DISPLAY -> ImageIcon SettingsSection.WEATHER -> Icons.Default.Refresh SettingsSection.ROUTINES -> Icons.Default.VolumeUp + SettingsSection.VOICE -> Icons.Default.Mic SettingsSection.APPS -> Icons.Default.Apps SettingsSection.ACCESS -> Icons.Default.Info } @@ -701,6 +727,14 @@ private fun SettingsScreen( refreshWeather ) SettingsSection.ROUTINES -> RoutineSettings(state.settings, updateSettings) + SettingsSection.VOICE -> VoiceSettings( + state = state, + setEnabled = setWakeWordEnabled, + setSensitivity = setWakeWordSensitivity, + pause = pauseWakeWord, + resume = resumeWakeWord, + test = testWakeWord + ) SettingsSection.APPS -> ApplicationsSettings(state, updateSettings) SettingsSection.ACCESS -> AccessSettings(requestHomeRole) } @@ -898,6 +932,124 @@ private fun RoutineSettings( } } +@Composable +private fun VoiceSettings( + state: LauncherUiState, + setEnabled: (Boolean) -> Unit, + setSensitivity: (Int) -> Unit, + pause: () -> Unit, + resume: () -> Unit, + test: () -> Unit +) = SettingsPage("Voice assistant") { + val context = LocalContext.current + val requiredPermissions = remember { + buildList { + add(Manifest.permission.RECORD_AUDIO) + if (Build.VERSION.SDK_INT >= 33) add(Manifest.permission.POST_NOTIFICATIONS) + }.toTypedArray() + } + val permissionLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.RequestMultiplePermissions() + ) { results -> + if (requiredPermissions.all { results[it] == true }) setEnabled(true) + } + fun permissionsGranted(): Boolean = requiredPermissions.all { + ContextCompat.checkSelfPermission(context, it) == PackageManager.PERMISSION_GRANTED + } + + SettingsCard( + "Listen for “Computer”", + "Detection stays on-device. Continued listening in other apps requires a persistent " + + "notification and Android’s microphone privacy indicator." + ) { + Switch( + checked = state.settings.wakeWord.enabled, + onCheckedChange = { enabled -> + if (!enabled) { + setEnabled(false) + } else if (permissionsGranted()) { + setEnabled(true) + } else { + permissionLauncher.launch(requiredPermissions) + } + } + ) + Text( + when (state.wakeWord.state) { + WakeWordState.DISABLED -> "Disabled" + WakeWordState.DOWNLOADING -> "Downloading model" + WakeWordState.INSTALLING -> "Installing model" + WakeWordState.READY -> "Ready" + WakeWordState.LISTENING -> "Listening" + WakeWordState.PAUSED -> "Paused" + WakeWordState.MICROPHONE_BUSY -> "Microphone busy; retrying" + WakeWordState.ERROR -> "Error" + }, + fontWeight = FontWeight.SemiBold + ) + if (state.wakeWord.message.isNotBlank()) { + Text(state.wakeWord.message, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + if ( + state.wakeWord.modelState == ModelInstallState.DOWNLOADING || + state.wakeWord.modelState == ModelInstallState.INSTALLING + ) { + LinearProgressIndicator( + progress = { state.wakeWord.installProgress / 100f }, + modifier = Modifier.fillMaxWidth() + ) + Text("${state.wakeWord.installProgress}%") + } + } + SettingsCard( + "Sensitivity", + "${state.settings.wakeWord.sensitivity}% · Higher values detect more easily but may false-trigger." + ) { + Slider( + value = state.settings.wakeWord.sensitivity.toFloat(), + onValueChange = { setSensitivity(it.roundToInt()) }, + valueRange = 0f..100f + ) + } + SettingsCard( + "Listening controls", + "Testing releases this launcher’s microphone before opening the selected Android assistant." + ) { + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + if (state.wakeWord.state == WakeWordState.PAUSED) { + Button(onClick = resume) { Text("Resume") } + } else { + OutlinedButton( + onClick = pause, + enabled = state.settings.wakeWord.enabled + ) { Text("Pause") } + } + Button( + onClick = test, + enabled = state.settings.wakeWord.enabled && + state.wakeWord.modelState == ModelInstallState.READY + ) { + Icon(Icons.Default.Mic, null) + Spacer(Modifier.width(7.dp)) + Text("Test assistant") + } + } + OutlinedButton( + onClick = { + runCatching { context.startActivity(Intent(Settings.ACTION_VOICE_INPUT_SETTINGS)) } + } + ) { + Text("Open Android voice settings") + } + } + SettingsCard( + "Privacy", + "Audio frames are processed locally and are never saved or transmitted." + ) { + Text("Only the last detection time and service status are retained for diagnostics.") + } +} + @Composable private fun ApplicationsSettings( state: LauncherUiState, @@ -929,10 +1081,24 @@ private fun AccessSettings(requestHomeRole: () -> Unit) = SettingsPage("System a PermissionStatus("Internet", true, "Used for weather and the built-in online photo collection.") PermissionStatus("Local photo folder", true, "Granted only to the folder you explicitly select.") PermissionStatus("Location", false, "Not requested. Weather uses coordinates entered in settings.") - PermissionStatus("Microphone", false, "Not requested. No wake-word model is bundled.") + PermissionStatus( + "Microphone", + ContextCompat.checkSelfPermission( + LocalContext.current, + Manifest.permission.RECORD_AUDIO + ) == PackageManager.PERMISSION_GRANTED, + "Requested only when on-device wake-word listening is enabled." + ) PermissionStatus("Display over other apps", false, "Not requested. Ultra-dim only covers this launcher.") PermissionStatus("Usage access", false, "Not requested. Recent-app history is not collected.") - PermissionStatus("Notifications", false, "Not requested because no background voice service runs.") + PermissionStatus( + "Notifications", + Build.VERSION.SDK_INT < 33 || ContextCompat.checkSelfPermission( + LocalContext.current, + Manifest.permission.POST_NOTIFICATIONS + ) == PackageManager.PERMISSION_GRANTED, + "Used for the visible listening service and assistant fallback action." + ) SettingsCard( "Audio output", "A normal launcher cannot force other apps to use a specific device." diff --git a/app/src/main/java/com/ambient/launcher/Models.kt b/app/src/main/java/com/ambient/launcher/Models.kt index 888e551..266e77b 100644 --- a/app/src/main/java/com/ambient/launcher/Models.kt +++ b/app/src/main/java/com/ambient/launcher/Models.kt @@ -36,6 +36,22 @@ data class DailyWeather( enum class KeepAwakeMode { WHILE_VISIBLE, WHILE_CHARGING, SYSTEM_DEFAULT } enum class ClockCorner { BOTTOM_LEFT, BOTTOM_RIGHT, TOP_LEFT, TOP_RIGHT } +enum class ModelInstallState { NOT_INSTALLED, DOWNLOADING, INSTALLING, READY, FAILED } +enum class WakeWordState { DISABLED, DOWNLOADING, INSTALLING, READY, LISTENING, PAUSED, MICROPHONE_BUSY, ERROR } + +data class WakeWordSettings( + val enabled: Boolean = false, + val sensitivity: Int = 50, + val installedModelVersion: String = "" +) + +data class WakeWordSnapshot( + val state: WakeWordState = WakeWordState.DISABLED, + val modelState: ModelInstallState = ModelInstallState.NOT_INSTALLED, + val installProgress: Int = 0, + val message: String = "", + val lastDetectionAt: Long? = null +) data class LauncherSettings( val intervalSeconds: Int = 60, @@ -53,7 +69,8 @@ data class LauncherSettings( val nightRoutineEnabled: Boolean = false, val nightStartHour: Int = 23, val nightEndHour: Int = 7, - val nightDimPercent: Int = 88 + val nightDimPercent: Int = 88, + val wakeWord: WakeWordSettings = WakeWordSettings() ) sealed interface Slide { diff --git a/app/src/main/java/com/ambient/launcher/SettingsStore.kt b/app/src/main/java/com/ambient/launcher/SettingsStore.kt index 5260f61..95e3967 100644 --- a/app/src/main/java/com/ambient/launcher/SettingsStore.kt +++ b/app/src/main/java/com/ambient/launcher/SettingsStore.kt @@ -29,6 +29,9 @@ class SettingsStore(private val context: Context) { val nightStart = intPreferencesKey("night_start") val nightEnd = intPreferencesKey("night_end") val nightDim = intPreferencesKey("night_dim") + val wakeEnabled = booleanPreferencesKey("wake_enabled") + val wakeSensitivity = intPreferencesKey("wake_sensitivity") + val wakeModelVersion = stringPreferencesKey("wake_model_version") } val settings: Flow = context.dataStore.data.map { p -> @@ -52,7 +55,12 @@ class SettingsStore(private val context: Context) { nightRoutineEnabled = p[Keys.nightEnabled] ?: false, nightStartHour = p[Keys.nightStart] ?: 23, nightEndHour = p[Keys.nightEnd] ?: 7, - nightDimPercent = p[Keys.nightDim] ?: 88 + nightDimPercent = p[Keys.nightDim] ?: 88, + wakeWord = WakeWordSettings( + enabled = p[Keys.wakeEnabled] ?: false, + sensitivity = p[Keys.wakeSensitivity] ?: 50, + installedModelVersion = p[Keys.wakeModelVersion] ?: "" + ) ) } @@ -74,9 +82,16 @@ class SettingsStore(private val context: Context) { p[Keys.nightStart] = value.nightStartHour p[Keys.nightEnd] = value.nightEndHour p[Keys.nightDim] = value.nightDimPercent + p[Keys.wakeEnabled] = value.wakeWord.enabled + p[Keys.wakeSensitivity] = value.wakeWord.sensitivity + p[Keys.wakeModelVersion] = value.wakeWord.installedModelVersion } } + suspend fun setWakeWordEnabled(enabled: Boolean) { + context.dataStore.edit { it[Keys.wakeEnabled] = enabled } + } + private inline fun > enumOrDefault(raw: String?, default: T): T = enumValues().firstOrNull { it.name == raw } ?: default } diff --git a/app/src/main/java/com/ambient/launcher/voice/DefaultAssistantLauncher.kt b/app/src/main/java/com/ambient/launcher/voice/DefaultAssistantLauncher.kt new file mode 100644 index 0000000..cb1b019 --- /dev/null +++ b/app/src/main/java/com/ambient/launcher/voice/DefaultAssistantLauncher.kt @@ -0,0 +1,37 @@ +package com.ambient.launcher.voice + +import android.app.PendingIntent +import android.content.ActivityNotFoundException +import android.content.Context +import android.content.Intent + +enum class AssistLaunchResult { LAUNCHED, NOT_CONFIGURED, BACKGROUND_BLOCKED } + +class DefaultAssistantLauncher(private val context: Context) { + fun launch(): AssistLaunchResult { + val intent = assistantIntent() + if (intent.resolveActivity(context.packageManager) == null) { + return AssistLaunchResult.NOT_CONFIGURED + } + return try { + context.startActivity(intent) + AssistLaunchResult.LAUNCHED + } catch (_: ActivityNotFoundException) { + AssistLaunchResult.NOT_CONFIGURED + } catch (_: SecurityException) { + AssistLaunchResult.BACKGROUND_BLOCKED + } catch (_: RuntimeException) { + AssistLaunchResult.BACKGROUND_BLOCKED + } + } + + fun pendingIntent(): PendingIntent = PendingIntent.getActivity( + context, + 8031, + assistantIntent(), + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + ) + + private fun assistantIntent() = Intent(Intent.ACTION_ASSIST) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP) +} diff --git a/app/src/main/java/com/ambient/launcher/voice/WakeWordController.kt b/app/src/main/java/com/ambient/launcher/voice/WakeWordController.kt new file mode 100644 index 0000000..7e92fdd --- /dev/null +++ b/app/src/main/java/com/ambient/launcher/voice/WakeWordController.kt @@ -0,0 +1,181 @@ +package com.ambient.launcher.voice + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Build +import androidx.core.content.ContextCompat +import androidx.work.Constraints +import androidx.work.ExistingWorkPolicy +import androidx.work.NetworkType +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.WorkInfo +import androidx.work.WorkManager +import com.ambient.launcher.ModelInstallState +import com.ambient.launcher.WakeWordSnapshot +import com.ambient.launcher.WakeWordState +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +object WakeWordRuntime { + private val mutable = MutableStateFlow(WakeWordSnapshot()) + val snapshot: StateFlow = mutable.asStateFlow() + + fun update( + state: WakeWordState? = null, + modelState: ModelInstallState? = null, + progress: Int? = null, + message: String? = null, + lastDetectionAt: Long? = null + ) { + val old = mutable.value + mutable.value = old.copy( + state = state ?: old.state, + modelState = modelState ?: old.modelState, + installProgress = progress ?: old.installProgress, + message = message ?: old.message, + lastDetectionAt = lastDetectionAt ?: old.lastDetectionAt + ) + } +} + +class WakeWordController(private val context: Context) { + val status: StateFlow = WakeWordRuntime.snapshot + private var enabled = false + private var visible = false + private var sensitivity = 50 + + init { + val lastDetection = context.getSharedPreferences("wake_runtime", Context.MODE_PRIVATE) + .getLong("last_detection_at", 0L) + .takeIf { it > 0L } + if (lastDetection != null) { + WakeWordRuntime.update(lastDetectionAt = lastDetection) + } + if (WakeWordModel.isReady(context)) { + WakeWordRuntime.update( + modelState = ModelInstallState.READY, + state = WakeWordState.READY, + progress = 100, + message = "Offline model ready" + ) + } + WorkManager.getInstance(context) + .getWorkInfosForUniqueWorkLiveData(WakeWordModel.WORK_NAME) + .observeForever { work -> + val latest = work.maxByOrNull { it.runAttemptCount } ?: return@observeForever + when (latest.state) { + WorkInfo.State.ENQUEUED, WorkInfo.State.BLOCKED -> WakeWordRuntime.update( + modelState = ModelInstallState.DOWNLOADING, + state = WakeWordState.DOWNLOADING, + message = "Waiting for a network connection" + ) + WorkInfo.State.RUNNING -> WakeWordRuntime.update( + progress = latest.progress.getInt( + "progress", + WakeWordRuntime.snapshot.value.installProgress + ) + ) + WorkInfo.State.SUCCEEDED -> { + WakeWordRuntime.update( + modelState = ModelInstallState.READY, + state = WakeWordState.READY, + progress = 100, + message = "Offline model ready" + ) + if (enabled && visible) startFromVisibleActivity() + } + WorkInfo.State.FAILED, WorkInfo.State.CANCELLED -> WakeWordRuntime.update( + modelState = ModelInstallState.FAILED, + state = WakeWordState.ERROR, + message = latest.outputData.getString("error") ?: "Model installation failed" + ) + } + } + } + + fun configure(isEnabled: Boolean, newSensitivity: Int) { + enabled = isEnabled + sensitivity = newSensitivity.coerceIn(0, 100) + if (!enabled) disable() + else if (visible) enable() + } + + fun setVisible(isVisible: Boolean) { + visible = isVisible + if (visible && enabled) enable() + } + + fun enable() { + enabled = true + if (!WakeWordModel.isReady(context)) { + enqueueModelInstall() + return + } + if (visible) startFromVisibleActivity() + } + + fun disable() { + enabled = false + context.stopService(Intent(context, WakeWordService::class.java)) + WakeWordRuntime.update(state = WakeWordState.DISABLED, message = "Wake word disabled") + } + + fun pause() = sendAction(WakeWordService.ACTION_PAUSE) + fun resume() = if (visible) sendAction(WakeWordService.ACTION_RESUME) else Unit + fun testDetection() = sendAction(WakeWordService.ACTION_TEST) + + fun startFromVisibleActivity() { + if (!enabled || !visible || !WakeWordModel.isReady(context)) return + if (!hasPermissions()) { + WakeWordRuntime.update( + state = WakeWordState.ERROR, + message = "Microphone and notification permission are required" + ) + return + } + val intent = Intent(context, WakeWordService::class.java) + .setAction(WakeWordService.ACTION_START) + .putExtra(WakeWordService.EXTRA_SENSITIVITY, sensitivity) + runCatching { ContextCompat.startForegroundService(context, intent) } + .onFailure { + WakeWordRuntime.update( + state = WakeWordState.ERROR, + message = "Open the launcher to restart listening" + ) + } + } + + private fun sendAction(action: String) { + val intent = Intent(context, WakeWordService::class.java).setAction(action) + runCatching { ContextCompat.startForegroundService(context, intent) } + } + + private fun enqueueModelInstall() { + val request = OneTimeWorkRequestBuilder() + .setConstraints( + Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build() + ) + .build() + WorkManager.getInstance(context).enqueueUniqueWork( + WakeWordModel.WORK_NAME, + ExistingWorkPolicy.KEEP, + request + ) + } + + private fun hasPermissions(): Boolean { + val microphone = ContextCompat.checkSelfPermission( + context, + Manifest.permission.RECORD_AUDIO + ) == PackageManager.PERMISSION_GRANTED + val notifications = Build.VERSION.SDK_INT < 33 || + ContextCompat.checkSelfPermission( + context, + Manifest.permission.POST_NOTIFICATIONS + ) == PackageManager.PERMISSION_GRANTED + return microphone && notifications + } +} diff --git a/app/src/main/java/com/ambient/launcher/voice/WakeWordModel.kt b/app/src/main/java/com/ambient/launcher/voice/WakeWordModel.kt new file mode 100644 index 0000000..9bf2001 --- /dev/null +++ b/app/src/main/java/com/ambient/launcher/voice/WakeWordModel.kt @@ -0,0 +1,198 @@ +package com.ambient.launcher.voice + +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.Data +import androidx.work.WorkerParameters +import com.ambient.launcher.ModelInstallState +import com.ambient.launcher.WakeWordState +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream +import java.io.BufferedInputStream +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.security.MessageDigest + +object WakeWordModel { + const val VERSION = "gigaspeech-3.3M-2024-01-01" + const val WORK_NAME = "install-wake-word-model" + const val DOWNLOAD_URL = + "https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/" + + "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz2" + const val ARCHIVE_SHA256 = "f170013b4716e41b62b9bfd809687c207cef798ef9bc6534d524e17af9b6561a" + private const val ROOT = "wake-word-model" + + val requiredFiles = setOf( + "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx", + "decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx", + "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx", + "tokens.txt", + "bpe.model" + ) + + fun directory(context: Context) = File(context.filesDir, ROOT) + fun isReady(context: Context): Boolean { + val dir = directory(context) + return File(dir, ".ready").readTextOrNull() == VERSION && + requiredFiles.all { File(dir, it).isFile } && + File(dir, "keywords.txt").isFile + } + + fun path(context: Context, name: String) = File(directory(context), name).absolutePath + + private fun File.readTextOrNull(): String? = runCatching { readText() }.getOrNull() +} + +class WakeWordModelWorker( + appContext: Context, + params: WorkerParameters +) : CoroutineWorker(appContext, params) { + override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + val parent = applicationContext.filesDir + val archive = File(parent, "wake-word-model.download") + val staging = File(parent, "wake-word-model.staging") + try { + WakeWordRuntime.update( + modelState = ModelInstallState.DOWNLOADING, + state = WakeWordState.DOWNLOADING, + message = "Downloading offline wake-word model" + ) + download(archive) + if (sha256(archive) != WakeWordModel.ARCHIVE_SHA256) { + error("Downloaded model failed integrity verification") + } + WakeWordRuntime.update( + modelState = ModelInstallState.INSTALLING, + state = WakeWordState.INSTALLING, + message = "Installing offline wake-word model" + ) + staging.deleteRecursively() + staging.mkdirs() + extractSelected(archive, staging) + createComputerKeyword(File(staging, "tokens.txt"), File(staging, "keywords.txt")) + if (!WakeWordModel.requiredFiles.all { File(staging, it).isFile }) { + error("Model archive is incomplete") + } + File(staging, ".ready").writeText(WakeWordModel.VERSION) + val destination = WakeWordModel.directory(applicationContext) + destination.deleteRecursively() + if (!staging.renameTo(destination)) error("Unable to activate installed model") + archive.delete() + WakeWordRuntime.update( + modelState = ModelInstallState.READY, + state = WakeWordState.READY, + progress = 100, + message = "Offline model ready" + ) + Result.success(Data.Builder().putString("version", WakeWordModel.VERSION).build()) + } catch (error: Throwable) { + archive.delete() + staging.deleteRecursively() + WakeWordRuntime.update( + modelState = ModelInstallState.FAILED, + state = WakeWordState.ERROR, + message = error.message ?: "Model installation failed" + ) + Result.failure(Data.Builder().putString("error", error.message).build()) + } + } + + private fun download(target: File) { + val connection = URL(WakeWordModel.DOWNLOAD_URL).openConnection() as HttpURLConnection + connection.connectTimeout = 15_000 + connection.readTimeout = 30_000 + connection.instanceFollowRedirects = true + connection.setRequestProperty("Accept", "application/octet-stream") + try { + if (connection.responseCode !in 200..299) { + error("Model download returned HTTP ${connection.responseCode}") + } + val total = connection.contentLengthLong.coerceAtLeast(1L) + var received = 0L + var lastProgress = -1 + connection.inputStream.buffered().use { input -> + target.outputStream().buffered().use { output -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + if (isStopped) error("Model download cancelled") + val count = input.read(buffer) + if (count < 0) break + output.write(buffer, 0, count) + received += count + val progress = ((received * 100L) / total).toInt().coerceIn(0, 99) + if (progress != lastProgress) { + lastProgress = progress + setProgressAsync(Data.Builder().putInt("progress", progress).build()) + WakeWordRuntime.update(progress = progress) + } + } + } + } + } finally { + connection.disconnect() + } + } + + private fun extractSelected(archive: File, destination: File) { + TarArchiveInputStream( + BZip2CompressorInputStream(BufferedInputStream(archive.inputStream())) + ).use { tar -> + while (true) { + val entry = tar.nextTarEntry ?: break + if (!entry.isFile) continue + val name = entry.name.substringAfterLast('/') + if (name !in WakeWordModel.requiredFiles) continue + val output = File(destination, name) + output.outputStream().buffered().use { tar.copyTo(it) } + } + } + } + + private fun createComputerKeyword(tokensFile: File, output: File) { + val tokens = tokensFile.readLines().mapIndexedNotNull { index, line -> + val columns = line.trim().split(Regex("\\s+")) + val piece = columns.firstOrNull()?.takeIf { it.isNotBlank() } ?: return@mapIndexedNotNull null + piece to (columns.getOrNull(1)?.toIntOrNull() ?: index) + } + val target = "▁COMPUTER" + data class Candidate(val pieces: List>, val score: Int) + val best = arrayOfNulls(target.length + 1) + best[0] = Candidate(emptyList(), 0) + for (start in target.indices) { + val current = best[start] ?: continue + tokens.forEach { token -> + val piece = token.first + if (target.startsWith(piece, start)) { + val end = start + piece.length + val candidate = Candidate(current.pieces + token, current.score + token.second) + val previous = best[end] + if ( + previous == null || + candidate.pieces.size < previous.pieces.size || + candidate.pieces.size == previous.pieces.size && candidate.score < previous.score + ) { + best[end] = candidate + } + } + } + } + val pieces = best[target.length]?.pieces ?: error("Model cannot tokenize COMPUTER") + output.writeText(pieces.joinToString(" ") { it.first } + "\n") + } + + private fun sha256(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().buffered().use { input -> + val buffer = ByteArray(DEFAULT_BUFFER_SIZE) + while (true) { + val count = input.read(buffer) + if (count < 0) break + digest.update(buffer, 0, count) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } + } +} diff --git a/app/src/main/java/com/ambient/launcher/voice/WakeWordService.kt b/app/src/main/java/com/ambient/launcher/voice/WakeWordService.kt new file mode 100644 index 0000000..678338b --- /dev/null +++ b/app/src/main/java/com/ambient/launcher/voice/WakeWordService.kt @@ -0,0 +1,354 @@ +package com.ambient.launcher.voice + +import android.Manifest +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.app.Service +import android.content.Intent +import android.content.pm.PackageManager +import android.content.pm.ServiceInfo +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.os.Build +import android.os.Handler +import android.os.IBinder +import android.os.Looper +import androidx.core.app.NotificationCompat +import androidx.core.app.ServiceCompat +import androidx.core.content.ContextCompat +import com.ambient.launcher.MainActivity +import com.ambient.launcher.R +import com.ambient.launcher.SettingsStore +import com.ambient.launcher.WakeWordState +import com.k2fsa.sherpa.onnx.FeatureConfig +import com.k2fsa.sherpa.onnx.KeywordSpotter +import com.k2fsa.sherpa.onnx.KeywordSpotterConfig +import com.k2fsa.sherpa.onnx.OnlineModelConfig +import com.k2fsa.sherpa.onnx.OnlineStream +import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlin.concurrent.thread + +class WakeWordService : Service() { + private val mainHandler = Handler(Looper.getMainLooper()) + private val serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private var recorder: AudioRecord? = null + private var spotter: KeywordSpotter? = null + private var stream: OnlineStream? = null + private var recordingThread: Thread? = null + @Volatile private var listening = false + @Volatile private var paused = false + private var sensitivity = 50 + private var lastDetection = 0L + + override fun onCreate() { + super.onCreate() + createChannels() + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + promoteToForeground() + when (intent?.action) { + ACTION_START -> { + val requestedSensitivity = + intent.getIntExtra(EXTRA_SENSITIVITY, 50).coerceIn(0, 100) + if (requestedSensitivity != sensitivity && listening) stopListening() + sensitivity = requestedSensitivity + paused = false + startListening() + } + ACTION_PAUSE -> pauseListening() + ACTION_RESUME -> { + paused = false + startListening() + } + ACTION_STOP -> { + stopListening() + stopForeground(STOP_FOREGROUND_REMOVE) + serviceScope.launch { + SettingsStore(applicationContext).setWakeWordEnabled(false) + stopSelf() + } + } + ACTION_TEST -> handleDetection(force = true) + else -> { + WakeWordRuntime.update( + state = WakeWordState.ERROR, + message = "Listening must be started from the visible launcher" + ) + stopSelf() + } + } + return START_STICKY + } + + override fun onDestroy() { + stopListening() + mainHandler.removeCallbacksAndMessages(null) + serviceScope.cancel() + super.onDestroy() + } + + override fun onBind(intent: Intent?): IBinder? = null + + private fun startListening() { + if (listening || paused) return + if (!WakeWordModel.isReady(this)) { + WakeWordRuntime.update(state = WakeWordState.ERROR, message = "Wake-word model is not ready") + return + } + if ( + ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) != + PackageManager.PERMISSION_GRANTED + ) { + WakeWordRuntime.update(state = WakeWordState.ERROR, message = "Microphone permission is required") + return + } + runCatching { + val minimum = AudioRecord.getMinBufferSize( + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT + ) + if (minimum <= 0) error("Microphone is unavailable") + val audioRecord = AudioRecord( + MediaRecorder.AudioSource.VOICE_RECOGNITION, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + minimum * 2 + ) + if (audioRecord.state != AudioRecord.STATE_INITIALIZED) { + audioRecord.release() + error("Microphone is busy") + } + val model = createSpotter() + val onlineStream = model.createStream() + recorder = audioRecord + spotter = model + stream = onlineStream + audioRecord.startRecording() + if (audioRecord.recordingState != AudioRecord.RECORDSTATE_RECORDING) { + error("Microphone is busy") + } + listening = true + WakeWordRuntime.update( + state = WakeWordState.LISTENING, + message = "Listening for “Computer”" + ) + updateNotification() + recordingThread = thread(start = true, name = "WakeWordAudio") { + processAudio() + } + }.onFailure { + releaseAudio() + WakeWordRuntime.update( + state = WakeWordState.MICROPHONE_BUSY, + message = it.message ?: "Microphone is busy" + ) + if (!paused) mainHandler.postDelayed({ startListening() }, RETRY_DELAY_MS) + } + } + + private fun createSpotter(): KeywordSpotter { + val threshold = (0.45f - sensitivity * 0.003f).coerceIn(0.12f, 0.45f) + return KeywordSpotter( + config = KeywordSpotterConfig( + featConfig = FeatureConfig(sampleRate = SAMPLE_RATE, featureDim = 80, dither = 0f), + modelConfig = OnlineModelConfig( + transducer = OnlineTransducerModelConfig( + encoder = WakeWordModel.path( + this, + "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx" + ), + decoder = WakeWordModel.path( + this, + "decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx" + ), + joiner = WakeWordModel.path( + this, + "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx" + ) + ), + tokens = WakeWordModel.path(this, "tokens.txt"), + numThreads = 1, + provider = "cpu", + modelType = "zipformer2" + ), + keywordsFile = WakeWordModel.path(this, "keywords.txt"), + keywordsScore = 1.5f, + keywordsThreshold = threshold, + maxActivePaths = 4, + numTrailingBlanks = 2 + ) + ) + } + + private fun processAudio() { + val buffer = ShortArray(1600) + while (listening) { + val count = recorder?.read(buffer, 0, buffer.size) ?: break + if (count <= 0) continue + val samples = FloatArray(count) { buffer[it] / 32768f } + val activeStream = stream ?: break + val activeSpotter = spotter ?: break + activeStream.acceptWaveform(samples, SAMPLE_RATE) + while (activeSpotter.isReady(activeStream)) { + activeSpotter.decode(activeStream) + if (activeSpotter.getResult(activeStream).keyword.isNotBlank()) { + activeSpotter.reset(activeStream) + handleDetection(force = false) + return + } + } + } + } + + private fun handleDetection(force: Boolean) { + val now = System.currentTimeMillis() + if (!force && now - lastDetection < DETECTION_DEBOUNCE_MS) return + lastDetection = now + getSharedPreferences("wake_runtime", MODE_PRIVATE) + .edit() + .putLong("last_detection_at", now) + .apply() + stopListening() + WakeWordRuntime.update( + state = WakeWordState.READY, + message = "Wake word detected", + lastDetectionAt = now + ) + val launcher = DefaultAssistantLauncher(this) + val result = launcher.launch() + if (result != AssistLaunchResult.LAUNCHED) showAssistantFallback(launcher) + if (!paused) mainHandler.postDelayed({ startListening() }, ASSISTANT_COOLDOWN_MS) + } + + private fun pauseListening() { + paused = true + stopListening() + WakeWordRuntime.update(state = WakeWordState.PAUSED, message = "Wake-word listening paused") + updateNotification() + } + + private fun stopListening() { + listening = false + runCatching { recorder?.stop() } + releaseAudio() + } + + private fun releaseAudio() { + runCatching { recorder?.release() } + recorder = null + stream?.release() + stream = null + spotter?.release() + spotter = null + recordingThread = null + } + + private fun promoteToForeground() { + val type = if (Build.VERSION.SDK_INT >= 30) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } else 0 + ServiceCompat.startForeground( + this, + LISTENING_NOTIFICATION_ID, + listeningNotification(), + type + ) + } + + private fun updateNotification() { + getSystemService(NotificationManager::class.java) + .notify(LISTENING_NOTIFICATION_ID, listeningNotification()) + } + + private fun listeningNotification(): Notification { + val openLauncher = PendingIntent.getActivity( + this, + 8010, + Intent(this, MainActivity::class.java), + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + ) + val toggleAction = if (paused) ACTION_RESUME else ACTION_PAUSE + val toggleLabel = if (paused) "Resume" else "Pause" + return NotificationCompat.Builder(this, LISTENING_CHANNEL) + .setSmallIcon(R.drawable.ic_launcher) + .setContentTitle(if (paused) "Wake word paused" else "Listening for “Computer”") + .setContentText("Tap to return to Ambient Launcher") + .setContentIntent(openLauncher) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setCategory(NotificationCompat.CATEGORY_SERVICE) + .addAction(0, toggleLabel, servicePendingIntent(toggleAction, 8011)) + .addAction(0, "Stop", servicePendingIntent(ACTION_STOP, 8012)) + .build() + } + + private fun showAssistantFallback(launcher: DefaultAssistantLauncher) { + val notification = NotificationCompat.Builder(this, ASSISTANT_CHANNEL) + .setSmallIcon(R.drawable.ic_launcher) + .setContentTitle("Wake word detected") + .setContentText("Tap to open your default assistant") + .setContentIntent(launcher.pendingIntent()) + .setAutoCancel(true) + .setPriority(NotificationCompat.PRIORITY_HIGH) + .build() + getSystemService(NotificationManager::class.java) + .notify(ASSISTANT_NOTIFICATION_ID, notification) + } + + private fun servicePendingIntent(action: String, requestCode: Int): PendingIntent = + PendingIntent.getService( + this, + requestCode, + Intent(this, WakeWordService::class.java).setAction(action), + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + ) + + private fun createChannels() { + val manager = getSystemService(NotificationManager::class.java) + manager.createNotificationChannel( + NotificationChannel( + LISTENING_CHANNEL, + "Wake-word listening", + NotificationManager.IMPORTANCE_LOW + ).apply { + description = "Shown while Ambient Launcher listens for the local wake word" + setSound(null, null) + } + ) + manager.createNotificationChannel( + NotificationChannel( + ASSISTANT_CHANNEL, + "Assistant actions", + NotificationManager.IMPORTANCE_HIGH + ) + ) + } + + companion object { + const val ACTION_START = "com.ambient.launcher.voice.START" + const val ACTION_PAUSE = "com.ambient.launcher.voice.PAUSE" + const val ACTION_RESUME = "com.ambient.launcher.voice.RESUME" + const val ACTION_STOP = "com.ambient.launcher.voice.STOP" + const val ACTION_TEST = "com.ambient.launcher.voice.TEST" + const val EXTRA_SENSITIVITY = "sensitivity" + private const val SAMPLE_RATE = 16_000 + private const val DETECTION_DEBOUNCE_MS = 5_000L + private const val ASSISTANT_COOLDOWN_MS = 12_000L + private const val RETRY_DELAY_MS = 4_000L + private const val LISTENING_CHANNEL = "wake_word_listening" + private const val ASSISTANT_CHANNEL = "wake_word_assistant" + private const val LISTENING_NOTIFICATION_ID = 801 + private const val ASSISTANT_NOTIFICATION_ID = 802 + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index e9fc3a4..738c65c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,6 +11,14 @@ dependencyResolutionManagement { repositories { google() mavenCentral() + ivy { + name = "sherpaOnnxReleases" + url = uri("https://github.com/k2-fsa/sherpa-onnx/releases/download") + patternLayout { + artifact("v[revision]/[artifact]-[revision].[ext]") + } + metadataSources { artifact() } + } } }