diff --git a/README.md b/README.md index 928aed81..26c030cf 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ See the official [plugin documentation](https://www.appdevforall.org/codeonthego | [`ai-core/`](ai-core/) | Shared on-device LLM inference backend (bundled llama.cpp AAR) plus a Gemini API backend, exposed to other plugins as a runtime service. | | [`ai-assistant/`](ai-assistant/) | In-IDE AI chat assistant with tool calling; talks to `ai-core` for inference over local or Gemini models. | | [`flutter-template/`](flutter-template/) | Adds Flutter starter project templates (Basic, BLoC, Provider, GetX, Riverpod) to the New Project screen. | +| [`code-suggestions-plugin/`](code-suggestions-plugin/) | Inline ghost-text code completions powered by AI. | +| [`speech-to-text-plugin/`](speech-to-text-plugin/) | Voice-to-code: converts speech to code with AI generation. | +| [`vector-search-plugin/`](vector-search-plugin/) | Semantic code search using embeddings and vector similarity. | ## Building a plugin diff --git a/ai-assistant/README.md b/ai-assistant/README.md index 54a729ef..5a01ae6b 100644 --- a/ai-assistant/README.md +++ b/ai-assistant/README.md @@ -56,6 +56,22 @@ The build resolves `plugin-api.jar` from the repo-root `../libs/`. - `fragments/AiSettingsFragment.kt`, `viewmodel/AiSettingsViewModel.kt` — model/backend config - `tool/` — the agent tool-loop (executor, router, per-tool handlers, approval) +## Security + +- **Gemini API key at rest.** When you use the Gemini backend, the API key is + stored in this plugin's private `SharedPreferences` (`AgentSettings`). That + store is **app-sandboxed** — other apps cannot read it — but it is **not + encrypted at rest**, so it is recoverable on a rooted or otherwise compromised + device. Remove it any time from **AI Settings** (or `clearGeminiApiKey()`). + Encryption is deliberately not layered on here: the key is shared at runtime + with the sibling `ai-core` plugin (which performs the Gemini calls), and + `EncryptedSharedPreferences` would force both independent plugins to agree on a + master-key alias and crypto library version — a fragile cross-plugin coupling. + A host-provided secure-storage service is the correct long-term fix. +- **Gemini API key in transit.** The key is sent to Google as an `x-goog-api-key` + request header (and via the SDK on the chat path), never in a URL query string. +- **File tools are confined to the project root** — see the in-IDE help page. + ## License GPL-3.0 — same as AndroidIDE / CodeOnTheGo. diff --git a/ai-assistant/src/main/assets/icon_day.png b/ai-assistant/src/main/assets/icon_day.png index 4cb1dc35..5dfa673e 100644 Binary files a/ai-assistant/src/main/assets/icon_day.png and b/ai-assistant/src/main/assets/icon_day.png differ diff --git a/ai-assistant/src/main/assets/icon_night.png b/ai-assistant/src/main/assets/icon_night.png index 355a5c0e..75ed2c65 100644 Binary files a/ai-assistant/src/main/assets/icon_night.png and b/ai-assistant/src/main/assets/icon_night.png differ diff --git a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/AiAssistantPlugin.kt b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/AiAssistantPlugin.kt index 92cd5220..afa3c292 100644 --- a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/AiAssistantPlugin.kt +++ b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/AiAssistantPlugin.kt @@ -229,8 +229,6 @@ class AiAssistantPlugin : IPlugin, UIExtension, DocumentationExtension { } } - // Note: Encrypted Gemini API key migration handled by EncryptedPrefs - if (migratedCount > 0) { context.logger.info("Migrated $migratedCount settings from app to plugin") } else { diff --git a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/AiSettingsFragment.kt b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/AiSettingsFragment.kt index 7ed32d68..4cedacba 100644 --- a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/AiSettingsFragment.kt +++ b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/AiSettingsFragment.kt @@ -24,6 +24,11 @@ import java.util.Locale class AiSettingsFragment : DialogFragment() { + companion object { + /** FragmentResult key signalling the chat screen that settings were closed. */ + const val RESULT_SETTINGS_CLOSED = "ai_settings_closed" + } + private lateinit var viewModel: AiSettingsViewModel private lateinit var settingsToolbar: LinearLayout private lateinit var backButton: ImageButton @@ -77,6 +82,15 @@ class AiSettingsFragment : DialogFragment() { setupBackendSelector() } + override fun onDismiss(dialog: android.content.DialogInterface) { + super.onDismiss(dialog) + // This is a dialog, so the chat screen behind it never gets onResume when we close. + // Signal it to re-resolve the selected backend (routing + availability + label). + if (isAdded) { + parentFragmentManager.setFragmentResult(RESULT_SETTINGS_CLOSED, Bundle.EMPTY) + } + } + private fun initializeViewModel() { viewModel = ViewModelProvider( this, @@ -213,7 +227,7 @@ class AiSettingsFragment : DialogFragment() { if (path != null) { modelPathTextView.visibility = View.VISIBLE - val fileName = path.substringAfterLast("/") + val fileName = viewModel.getSavedModelName() ?: viewModel.fallbackDisplayName(path) modelPathTextView.text = "Saved: $fileName" } else { modelPathTextView.visibility = View.GONE @@ -387,11 +401,18 @@ class AiSettingsFragment : DialogFragment() { adapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item) modelSpinner.adapter = adapter - // Set current selection + // Set current selection. If the saved model is no longer offered + // migrate to the first available one so the next chat request + // doesn't 404 on a dead model name. val currentModel = viewModel.getGeminiModel() val currentIndex = models.indexOf(currentModel) if (currentIndex >= 0) { modelSpinner.setSelection(currentIndex) + } else if (models.isNotEmpty()) { + modelSpinner.setSelection(0) + val migrated = models[0] + viewModel.saveGeminiModel(migrated) + currentModelText?.text = "Current: $migrated" } } diff --git a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/ChatFragment.kt b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/ChatFragment.kt index cbabc4ba..2985a31d 100644 --- a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/ChatFragment.kt +++ b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/fragments/ChatFragment.kt @@ -86,6 +86,16 @@ class ChatFragment : Fragment() { setupBackendIndicator() observeViewModel() + // The settings screen is a DialogFragment, so this fragment's onResume does not fire + // when it closes. Listen for its dismissal to re-resolve the selected backend (routing + + // availability) and refresh the indicator label. + parentFragmentManager.setFragmentResultListener( + AiSettingsFragment.RESULT_SETTINGS_CLOSED, viewLifecycleOwner + ) { _, _ -> + viewModel.checkBackendAvailability() + viewModel.refreshBackendLabel() + } + // Check for test prompt from broadcast receiver (E2E testing) injectPendingTestPrompt() } @@ -139,6 +149,8 @@ class ChatFragment : Fragment() { // Check backend availability when fragment becomes visible // This ensures we check after all plugins have loaded viewModel.checkBackendAvailability() + // Reflect the currently selected backend (updates after returning from settings). + viewModel.refreshBackendLabel() } private fun initializeViewModel() { @@ -225,7 +237,13 @@ class ChatFragment : Fragment() { } private fun setupBackendIndicator() { - binding.backendStatusText.text = "Gemini" + viewLifecycleOwner.lifecycleScope.launch { + viewLifecycleOwner.repeatOnLifecycle(Lifecycle.State.STARTED) { + viewModel.activeBackendLabel.collect { label -> + binding.backendStatusText.text = label + } + } + } } private fun observeViewModel() { diff --git a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/AiSettingsViewModel.kt b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/AiSettingsViewModel.kt index b7ebf272..b3970915 100644 --- a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/AiSettingsViewModel.kt +++ b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/AiSettingsViewModel.kt @@ -42,6 +42,20 @@ class AiSettingsViewModel( private val getContext: () -> com.itsaky.androidide.plugins.PluginContext? ) : ViewModel() { + companion object { + private const val TAG = "AiSettingsViewModel" + + /** Default selection; kept in sync with GeminiBackend.DEFAULT_MODEL. */ + private const val DEFAULT_GEMINI_MODEL = "gemini-2.5-flash" + + /** Shown only when the live catalog can't be fetched — current models, no retired ones. */ + private val FALLBACK_MODELS = listOf( + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-2.0-flash", + ) + } + private val _savedModelPath = MutableLiveData(null) val savedModelPath: LiveData get() = _savedModelPath @@ -57,15 +71,59 @@ class AiSettingsViewModel( private fun checkInitialState() { val prefs = getPluginPrefs() - _savedModelPath.value = prefs?.getString("local_llm_model_path", null) + val savedPath = prefs?.getString("local_llm_model_path", null) + _savedModelPath.value = savedPath // For plugin, engine is always "ready" since it's managed by ai-core plugin _engineState.value = EngineState.Initialized - _modelLoadingState.value = ModelLoadingState.Idle + // Reflect a previously selected model so it survives closing/reopening settings, + // using the display name persisted at load time (no content-provider query here). + _modelLoadingState.value = if (savedPath != null) { + ModelLoadingState.Loaded(getSavedModelName() ?: fallbackDisplayName(savedPath)) + } else { + ModelLoadingState.Idle + } } private fun getPluginPrefs() = getContext()?.getPluginSharedPreferences("AgentSettings") + /** Human-readable name persisted alongside the model path at load time, if any. */ + fun getSavedModelName(): String? = + getPluginPrefs()?.getString("local_llm_model_name", null)?.takeIf { it.isNotBlank() } + + private fun saveLocalModelName(name: String?) { + getPluginPrefs()?.edit()?.putString("local_llm_model_name", name)?.apply() + } + + /** Decoded last path segment — a cheap fallback that at least avoids raw %3A escapes. */ + fun fallbackDisplayName(uriOrPath: String): String = + (try { android.net.Uri.decode(uriOrPath) } catch (e: Exception) { uriOrPath }).substringAfterLast('/') + + /** + * Resolve the real file name for a selected model. For a `content://` URI this queries the + * document provider's [OpenableColumns.DISPLAY_NAME] (e.g. "Llama-3.2-1B.gguf"); otherwise, + * and on any failure, it falls back to the decoded last path segment. Do NOT call on the main + * thread — the provider query can block. + */ + private fun resolveDisplayName(uriString: String): String { + if (uriString.startsWith("content://")) { + try { + val uri = android.net.Uri.parse(uriString) + getContext()?.androidContext?.contentResolver + ?.query(uri, arrayOf(android.provider.OpenableColumns.DISPLAY_NAME), null, null, null) + ?.use { c -> + val idx = c.getColumnIndex(android.provider.OpenableColumns.DISPLAY_NAME) + if (idx >= 0 && c.moveToFirst() && !c.isNull(idx)) { + c.getString(idx)?.takeIf { it.isNotBlank() }?.let { return it } + } + } + } catch (e: Exception) { + android.util.Log.w(TAG, "Could not resolve display name for $uriString", e) + } + } + return fallbackDisplayName(uriString) + } + fun getAvailableBackends(): List = AiBackend.entries fun saveBackend(backend: AiBackend) { @@ -120,6 +178,12 @@ class AiSettingsViewModel( return getPluginPrefs()?.getBoolean("use_simple_local_prompt", true) ?: true } + /** + * Persists the Gemini API key in this plugin's private SharedPreferences. + * The store is app-sandboxed (not world-readable) but NOT encrypted at rest; + * it is recoverable on a rooted/compromised device. Use [clearGeminiApiKey] + * to remove it. See the plugin README "Security" section for the tradeoff. + */ fun saveGeminiApiKey(apiKey: String) { getPluginPrefs()?.edit()?.apply { putString("gemini_api_key", apiKey) @@ -152,7 +216,7 @@ class AiSettingsViewModel( } fun getGeminiModel(): String { - return getPluginPrefs()?.getString("gemini_model", "gemini-1.5-flash") ?: "gemini-1.5-flash" + return getPluginPrefs()?.getString("gemini_model", DEFAULT_GEMINI_MODEL) ?: DEFAULT_GEMINI_MODEL } private val _geminiModels = MutableLiveData>(emptyList()) @@ -161,77 +225,52 @@ class AiSettingsViewModel( private val _geminiModelsLoading = MutableLiveData(false) val geminiModelsLoading: LiveData get() = _geminiModelsLoading + /** + * Ask ai-core's Gemini backend for the models the current API key can actually + * use, and publish them to [geminiModels]. Falls back to [FALLBACK_MODELS] (current + * models only — never a retired one) when there is no key, no backend, or the live + * lookup fails, so the picker is never populated with a model that would 404. + */ fun fetchGeminiModels() { viewModelScope.launch(Dispatchers.IO) { _geminiModelsLoading.postValue(true) try { - // Get the LlmInferenceService from SharedServices - val llmService = SharedServices.get(LlmInferenceService::class.java) - if (llmService == null) { - android.util.Log.e("AiSettingsViewModel", "LlmInferenceService not available") - _geminiModels.postValue(listOf( - "gemini-1.5-flash", - "gemini-1.5-pro", - "gemini-2.5-flash", - "gemini-2.5-pro", - "gemini-3-flash", - "gemini-3.5-flash" - )) - _geminiModelsLoading.postValue(false) + val apiKey = getGeminiApiKey()?.trim() + if (apiKey.isNullOrBlank()) { + android.util.Log.w(TAG, "No Gemini API key saved; showing fallback models") + _geminiModels.postValue(FALLBACK_MODELS) return@launch } - // Get the Gemini backend - val geminiBackend = llmService.getBackend("gemini") + val llmService = SharedServices.get(LlmInferenceService::class.java) + val geminiBackend = llmService?.getBackend("gemini") if (geminiBackend == null) { - android.util.Log.e("AiSettingsViewModel", "Gemini backend not available") - _geminiModels.postValue(listOf( - "gemini-1.5-flash", - "gemini-1.5-pro", - "gemini-2.5-flash", - "gemini-2.5-pro", - "gemini-3-flash", - "gemini-3.5-flash" - )) - _geminiModelsLoading.postValue(false) + android.util.Log.e(TAG, "Gemini backend not available") + _geminiModels.postValue(FALLBACK_MODELS) return@launch } - // Try to get list models method via reflection - // (since GeminiBackend is not in the interface) - try { - val listModelsMethod = geminiBackend.javaClass.getMethod("listModels") - val futureResult = listModelsMethod.invoke(geminiBackend) - - if (futureResult is java.util.concurrent.CompletableFuture<*>) { - @Suppress("UNCHECKED_CAST") - val models = (futureResult as java.util.concurrent.CompletableFuture>).get() - android.util.Log.d("AiSettingsViewModel", "Fetched ${models.size} Gemini models") - _geminiModels.postValue(models) - } - } catch (e: Exception) { - android.util.Log.e("AiSettingsViewModel", "Error fetching models", e) - // Fallback to default list - _geminiModels.postValue(listOf( - "gemini-1.5-flash", - "gemini-1.5-pro", - "gemini-2.5-flash", - "gemini-2.5-pro", - "gemini-3-flash", - "gemini-3.5-flash" - )) + // listModels() lives in the ai-core plugin and isn't part of the shared + // LlmBackend interface, so reach it reflectively across the plugin + // classloader boundary. It reads the saved key itself and returns the live + // v1beta catalog (empty when unavailable). + val method = geminiBackend.javaClass.getMethod("listModels") + val futureResult = method.invoke(geminiBackend) + + @Suppress("UNCHECKED_CAST") + val models: List = + (futureResult as? java.util.concurrent.CompletableFuture>)?.get().orEmpty() + if (models.isEmpty()) { + android.util.Log.w(TAG, "Live model list empty; showing fallback models") + _geminiModels.postValue(FALLBACK_MODELS) + } else { + android.util.Log.d(TAG, "Fetched ${models.size} Gemini models") + _geminiModels.postValue(models) } } catch (e: Exception) { - android.util.Log.e("AiSettingsViewModel", "Error in fetchGeminiModels", e) - _geminiModels.postValue(listOf( - "gemini-1.5-flash", - "gemini-1.5-pro", - "gemini-2.5-flash", - "gemini-2.5-pro", - "gemini-3-flash", - "gemini-3.5-flash" - )) + android.util.Log.e(TAG, "Error fetching Gemini models", e) + _geminiModels.postValue(FALLBACK_MODELS) } finally { _geminiModelsLoading.postValue(false) } @@ -248,10 +287,11 @@ class AiSettingsViewModel( _modelLoadingState.postValue(ModelLoadingState.Loading) try { - // Extract filename from URI for display - val fileName = uriString.substringAfterLast("/") + // Resolve the real file name (not the raw content-URI doc id) for display. + val fileName = resolveDisplayName(uriString) - // Save the path + // Persist the name before the path so the savedModelPath observer can read it. + saveLocalModelName(fileName) saveLocalModelPath(uriString) // In plugin context, we don't directly load the model @@ -260,7 +300,7 @@ class AiSettingsViewModel( ModelLoadingState.Loaded(fileName) ) - android.util.Log.d("AiSettingsViewModel", "Model path saved: $uriString") + android.util.Log.d(TAG, "Model path saved: $uriString ($fileName)") } catch (e: Exception) { android.util.Log.e("AiSettingsViewModel", "Error saving model path", e) _modelLoadingState.postValue( diff --git a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/ChatViewModel.kt b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/ChatViewModel.kt index 0102ed38..af00120a 100644 --- a/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/ChatViewModel.kt +++ b/ai-assistant/src/main/kotlin/com/itsaky/androidide/plugins/aiassistant/viewmodel/ChatViewModel.kt @@ -85,6 +85,29 @@ class ChatViewModel( private var currentBackendId: String = "local" // Default to local backend + /** + * Human-readable label for the backend the user has *selected* in settings, shown under the + * chat input. This tracks the selection (the `ai_backend_preference`), NOT the + * availability-resolved backend — selecting Gemini must read "Gemini API" even before its + * API key check runs, otherwise it would always fall back to "Local LLM". + */ + private val _activeBackendLabel = MutableStateFlow(selectedBackendLabel()) + val activeBackendLabel: StateFlow = _activeBackendLabel.asStateFlow() + + private fun selectedBackendLabel(): String { + val pref = getContext()?.getPluginSharedPreferences("AgentSettings") + ?.getString("ai_backend_preference", "LOCAL_LLM") + return when (pref) { + "GEMINI" -> "Gemini API" + else -> "Local LLM" + } + } + + /** Re-read the selected backend and update [activeBackendLabel]; call when returning to chat. */ + fun refreshBackendLabel() { + _activeBackendLabel.value = selectedBackendLabel() + } + // Tool execution infrastructure private val approvalManager = ToolApprovalManager() private val toolRouter: ToolRouter diff --git a/ai-assistant/src/main/res/layout/fragment_ai_settings.xml b/ai-assistant/src/main/res/layout/fragment_ai_settings.xml index 40892ed0..421e3f68 100644 --- a/ai-assistant/src/main/res/layout/fragment_ai_settings.xml +++ b/ai-assistant/src/main/res/layout/fragment_ai_settings.xml @@ -29,7 +29,7 @@ android:layout_weight="1" android:text="AI Settings" android:textAppearance="?android:attr/textAppearanceLarge" - android:textColor="?android:attr/textColorPrimaryInverse" /> + android:textColor="?android:attr/textColorPrimary" /> + + + + +AI Core — Guide + + + +

AI Core — Inference Backend

+ +

AI Core is a headless plugin: it has no screens of its own. It + provides the shared LLM inference service that the other AI plugins + (AI Assistant, Code Suggestions, Speech to Text, + Vector Search) call at runtime. Install AI Core first, then any + of the plugins that depend on it.

+ +

Backends

+
    +
  • Local (on-device) — runs a .gguf model through a + bundled, prebuilt llama.cpp library. Nothing leaves the device.
    • +
  • Gemini (cloud) — calls Google's Gemini API over HTTPS. Requires a + Gemini API key, entered in AI Assistant → AI Settings. The key is + sent as an x-goog-api-key header, never in a URL.
    • +
+ +

Choosing / configuring a model

+

AI Core has no settings screen of its own — the model file, backend + choice, and Gemini API key are all configured from AI Assistant → AI + Settings. AI Core reads that configuration at request time.

+ +
+ Embedding-only .gguf files cannot serve chat requests. If you + select an embedding model for chat, requests are rejected with a clear error + instead of crashing. +
+ +

Privacy

+
    +
  • Local backend — prompts and code never leave the device.
    • +
  • Gemini backend — prompts and any file contents a plugin sends are + transmitted to Google over HTTPS.
    • +
+ +

Troubleshooting

+
    +
  • "LlmInferenceService not available" in another plugin — AI Core + isn't installed/activated, or the IDE needs a restart.
    • +
  • "No model configured" — pick a .gguf file (Local) or + set an API key (Gemini) in AI Assistant's settings.
    • +
  • Gemini errors — verify the API key, network connection, and quota.
    • +
+ + diff --git a/ai-core/src/main/assets/icon_day.png b/ai-core/src/main/assets/icon_day.png index 4cb1dc35..0102f74b 100644 Binary files a/ai-core/src/main/assets/icon_day.png and b/ai-core/src/main/assets/icon_day.png differ diff --git a/ai-core/src/main/assets/icon_night.png b/ai-core/src/main/assets/icon_night.png index 355a5c0e..070800ae 100644 Binary files a/ai-core/src/main/assets/icon_night.png and b/ai-core/src/main/assets/icon_night.png differ diff --git a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/AiCorePlugin.kt b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/AiCorePlugin.kt index 7f23d5b0..36a0f061 100644 --- a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/AiCorePlugin.kt +++ b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/AiCorePlugin.kt @@ -2,6 +2,9 @@ package com.itsaky.androidide.plugins.aicore import com.itsaky.androidide.plugins.IPlugin import com.itsaky.androidide.plugins.PluginContext +import com.itsaky.androidide.plugins.extensions.DocumentationExtension +import com.itsaky.androidide.plugins.extensions.PluginTooltipButton +import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry import com.itsaky.androidide.plugins.services.LlmInferenceService import com.itsaky.androidide.plugins.services.SharedServices @@ -9,7 +12,7 @@ import com.itsaky.androidide.plugins.services.SharedServices * AI Core Plugin providing LLM inference capabilities. * Implements LlmInferenceService and registers local LLM backend. */ -class AiCorePlugin : IPlugin { +class AiCorePlugin : IPlugin, DocumentationExtension { private lateinit var context: PluginContext private lateinit var llmService: LlmInferenceServiceImpl @@ -18,6 +21,7 @@ class AiCorePlugin : IPlugin { companion object { const val PLUGIN_ID = "com.itsaky.androidide.plugins.aicore" + private const val TOOLTIP_TAG_PLUGIN = "plugin_ai_core" } override fun initialize(context: PluginContext): Boolean { @@ -101,4 +105,30 @@ class AiCorePlugin : IPlugin { context.logger.info("AiCorePlugin: Released Gemini backend") } } + + override fun getTooltipCategory(): String = "plugin_ai_core" + + override fun getTooltipEntries(): List = listOf( + PluginTooltipEntry( + tag = TOOLTIP_TAG_PLUGIN, + summary = "AI Core provides the shared local and Gemini LLM backends used by the AI plugins.", + detail = """ +

AI Core is a headless plugin that registers the shared + LLM inference service used by AI Assistant, Code Suggestions, + Speech to Text, and Vector Search.

+

Configure the backend and model from AI Assistant → AI + Settings. Local models stay on-device; Gemini requests are + sent to Google over HTTPS.

+ """.trimIndent(), + buttons = listOf( + PluginTooltipButton( + description = "AI Core guide", + uri = "index.html", + order = 0 + ) + ) + ) + ) + + override fun getTier3DocsAssetPath(): String = "docs" } diff --git a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GeminiBackend.kt b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GeminiBackend.kt index 714aa8b7..1c95ba44 100644 --- a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GeminiBackend.kt +++ b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GeminiBackend.kt @@ -28,9 +28,20 @@ class GeminiBackend(private val context: PluginContext) : LlmBackend { @Volatile private var currentJob: Job? = null + companion object { + /** Current default model. gemini-1.5-* is retired on v1beta and now 404s. */ + const val DEFAULT_MODEL = "gemini-2.5-flash" + + /** ListModels endpoint for the same API version the SDK uses for generateContent. */ + private const val LIST_MODELS_URL = + "https://generativelanguage.googleapis.com/v1beta/models" + + /** Only models advertising this generation method can be used for chat. */ + private const val METHOD_GENERATE_CONTENT = "generateContent" + } + /** - * Get the model name from preferences, or use default. - * Default to gemini-1.5-flash for compatibility, fallback to gemini-2.5-flash. + * Get the model name from preferences, or use the current default. */ private fun getModelName(): String { val prefs = try { @@ -40,7 +51,19 @@ class GeminiBackend(private val context: PluginContext) : LlmBackend { context.logger.error("GeminiBackend: Error getting preferences", e) null } - return prefs?.getString("gemini_model", "gemini-1.5-flash") ?: "gemini-1.5-flash" + return prefs?.getString("gemini_model", DEFAULT_MODEL) ?: DEFAULT_MODEL + } + + /** Read the saved Gemini API key from ai-assistant's shared prefs, or null. */ + private fun readGeminiApiKey(): String? { + val prefs = try { + SharedServices.get(PluginContext::class.java) + ?.getPluginSharedPreferences("AgentSettings") + } catch (e: Exception) { + context.logger.error("GeminiBackend: Error getting preferences", e) + null + } + return prefs?.getString("gemini_api_key", null)?.trim()?.takeIf { it.isNotBlank() } } override fun getId(): String = "gemini" @@ -291,44 +314,94 @@ class GeminiBackend(private val context: PluginContext) : LlmBackend { } /** - * List available Gemini models. - * Returns a CompletableFuture with list of model names. + * List the Gemini models currently available to the saved API key that support chat + * (i.e. advertise the `generateContent` method). The result reflects the live v1beta + * catalog, so any name returned here is safe to pass to [generate] / [generateStreaming] + * without a 404 for a retired model. * - * Note: The Gemini Java SDK 1.16.0 doesn't have a direct list models API, - * so we return a curated list of commonly available models. + * The Gemini Java SDK 1.16.0 has no list-models call, so this queries the REST + * `ListModels` endpoint directly. Returns an empty list when no key is configured and + * completes exceptionally on a network/API failure, so the caller can fall back to a + * current-models-only list and never advertise a dead model. */ fun listModels(): CompletableFuture> { val future = CompletableFuture>() scope.launch { try { - context.logger.info("GeminiBackend: Returning available models list") - - // Return list of known Gemini models - // gemini-1.5-* may be deprecated, but included for fallback - future.complete(listOf( - "gemini-1.5-flash", - "gemini-1.5-pro", - "gemini-2.5-flash", - "gemini-2.5-flash-lite", - "gemini-2.5-pro", - "gemini-3-flash", - "gemini-3.5-flash" - )) + val key = readGeminiApiKey() + if (key.isNullOrBlank()) { + context.logger.warn("GeminiBackend: no API key configured; cannot list live models") + future.complete(emptyList()) + return@launch + } + val models = fetchAvailableModels(key) + context.logger.info("GeminiBackend: ${models.size} models support $METHOD_GENERATE_CONTENT") + future.complete(models) } catch (e: Exception) { context.logger.error("GeminiBackend: Error in listModels", e) - // Return minimal fallback list on error - future.complete(listOf( - "gemini-1.5-flash", - "gemini-2.5-flash", - "gemini-3.5-flash" - )) + future.completeExceptionally(e) } } return future } + /** + * Fetch and parse the ListModels catalog, following pagination, keeping only + * models that support [METHOD_GENERATE_CONTENT] and stripping the `models/` + * prefix from each name. Runs on the caller's (IO) coroutine. + */ + private fun fetchAvailableModels(apiKey: String): List { + val names = mutableListOf() + var pageToken: String? = null + + do { + val url = buildString { + append(LIST_MODELS_URL) + append("?pageSize=1000") + pageToken?.let { append("&pageToken=").append(java.net.URLEncoder.encode(it, "UTF-8")) } + } + + val conn = (java.net.URL(url).openConnection() as java.net.HttpURLConnection).apply { + requestMethod = "GET" + connectTimeout = 15_000 + readTimeout = 15_000 + // Pass the API key as a header, never in the URL query string: query + // strings leak into logs, proxies, and crash reports. + setRequestProperty("x-goog-api-key", apiKey) + } + + val body = try { + val code = conn.responseCode + if (code !in 200..299) { + val err = conn.errorStream?.bufferedReader()?.use { it.readText() }.orEmpty() + throw java.io.IOException("ListModels HTTP $code: $err") + } + conn.inputStream.bufferedReader().use { it.readText() } + } finally { + conn.disconnect() + } + + val json = org.json.JSONObject(body) + val models = json.optJSONArray("models") + if (models != null) { + for (i in 0 until models.length()) { + val model = models.getJSONObject(i) + val methods = model.optJSONArray("supportedGenerationMethods") ?: continue + val supportsChat = (0 until methods.length()) + .any { methods.optString(it) == METHOD_GENERATE_CONTENT } + if (!supportsChat) continue + val name = model.optString("name").removePrefix("models/") + if (name.isNotBlank()) names.add(name) + } + } + pageToken = json.optString("nextPageToken").takeIf { it.isNotBlank() } + } while (pageToken != null) + + return names.distinct() + } + /** * Create Gemini client with API key from settings. */ diff --git a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GgufModelInspector.kt b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GgufModelInspector.kt new file mode 100644 index 00000000..5a061469 --- /dev/null +++ b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/GgufModelInspector.kt @@ -0,0 +1,155 @@ +package com.itsaky.androidide.plugins.aicore + +import java.io.BufferedInputStream +import java.io.DataInputStream +import java.io.File +import java.io.FileInputStream + +/** + * Minimal GGUF header reader, just enough to tell whether a `.gguf` file is a chat/generation + * model or an embedding (encoder-only) model. + * + * WHY: the native chat path runs a causal `llama_decode`. Handed an encoder-only model (BERT + * family, e.g. all-MiniLM), llama.cpp hits a `GGML_ASSERT` and calls `abort()` — a SIGABRT that + * no Kotlin `try/catch` can intercept, taking the whole IDE process down. We classify the file + * up front so the backend can refuse chat gracefully instead of crashing. See ADFA-4388. + * + * This reads only the GGUF metadata header (`general.architecture` is almost always the first + * key), skipping over values without loading the model, so it's cheap. It deliberately + * **fails open**: any parse error or missing architecture is reported as [ModelKind.UNKNOWN] and + * treated as chat-capable, so a genuine chat model is never wrongly blocked by a header quirk. + */ +object GgufModelInspector { + + private const val GGUF_MAGIC = 0x46554747 // "GGUF" little-endian + + // GGUF metadata value types. + private const val T_UINT8 = 0 + private const val T_INT8 = 1 + private const val T_UINT16 = 2 + private const val T_INT16 = 3 + private const val T_UINT32 = 4 + private const val T_INT32 = 5 + private const val T_FLOAT32 = 6 + private const val T_BOOL = 7 + private const val T_STRING = 8 + private const val T_ARRAY = 9 + private const val T_UINT64 = 10 + private const val T_INT64 = 11 + private const val T_FLOAT64 = 12 + + /** + * Architectures that are encoder-only embedding models and cannot do causal generation. + * The `contains("bert")` catch below covers the whole BERT family (bert, nomic-bert, + * jina-bert-*, roberta, xlm-roberta, …); the explicit set covers the rest. + */ + private val EMBEDDING_ARCHS = setOf("mpnet", "gte", "t5encoder") + + enum class ModelKind { CHAT, EMBEDDING, UNKNOWN } + + data class Result(val kind: ModelKind, val architecture: String?) { + val isEmbeddingOnly: Boolean get() = kind == ModelKind.EMBEDDING + } + + /** Reads [modelPath]'s GGUF header and classifies it. Never throws. */ + fun classify(modelPath: String): Result { + val arch = try { + readArchitecture(File(modelPath)) + } catch (_: Exception) { + null + } ?: return Result(ModelKind.UNKNOWN, null) + + val a = arch.lowercase() + val isEmbedding = a.contains("bert") || a in EMBEDDING_ARCHS + return Result(if (isEmbedding) ModelKind.EMBEDDING else ModelKind.CHAT, arch) + } + + private fun readArchitecture(file: File): String? { + DataInputStream(BufferedInputStream(FileInputStream(file), 1 shl 16)).use { input -> + val magic = readU32(input) + if (magic != GGUF_MAGIC) return null + + val version = readU32(input) + // v1 used 32-bit counts/lengths; v2+ use 64-bit. + val wide = version >= 2 + + // tensor_count, then metadata_kv_count. + readCount(input, wide) + val kvCount = readCount(input, wide) + + for (i in 0 until kvCount) { + val key = readString(input, wide) + val valueType = readU32(input) + if (key == "general.architecture" && valueType == T_STRING) { + return readRawString(input, wide) + } + skipValue(input, valueType, wide) + } + } + return null + } + + private fun skipValue(input: DataInputStream, type: Int, wide: Boolean) { + when (type) { + T_UINT8, T_INT8, T_BOOL -> skipFully(input, 1) + T_UINT16, T_INT16 -> skipFully(input, 2) + T_UINT32, T_INT32, T_FLOAT32 -> skipFully(input, 4) + T_UINT64, T_INT64, T_FLOAT64 -> skipFully(input, 8) + T_STRING -> skipFully(input, readCount(input, wide)) + T_ARRAY -> { + val elemType = readU32(input) + val n = readCount(input, wide) + // GGUF arrays are never nested, so elements are scalars or strings. + repeat(n.toInt().coerceAtLeast(0)) { skipValue(input, elemType, wide) } + } + else -> throw IllegalStateException("Unknown GGUF value type: $type") + } + } + + // --- little-endian primitives --- + + private fun readU32(input: DataInputStream): Int { + val b0 = input.read(); val b1 = input.read(); val b2 = input.read(); val b3 = input.read() + if (b3 < 0) throw java.io.EOFException() + return (b0 and 0xFF) or ((b1 and 0xFF) shl 8) or ((b2 and 0xFF) shl 16) or ((b3 and 0xFF) shl 24) + } + + private fun readU64(input: DataInputStream): Long { + var v = 0L + for (i in 0 until 8) { + val b = input.read() + if (b < 0) throw java.io.EOFException() + v = v or ((b.toLong() and 0xFF) shl (8 * i)) + } + return v + } + + /** A length/count field: 64-bit on GGUF v2+, 32-bit on v1. */ + private fun readCount(input: DataInputStream, wide: Boolean): Long = + if (wide) readU64(input) else readU32(input).toLong() and 0xFFFFFFFFL + + private fun readString(input: DataInputStream, wide: Boolean): String = readRawString(input, wide) + + private fun readRawString(input: DataInputStream, wide: Boolean): String { + val len = readCount(input, wide) + // Keys/arch strings are tiny; guard against a corrupt huge length. + if (len < 0 || len > 1 shl 20) throw IllegalStateException("Unreasonable GGUF string length: $len") + val bytes = ByteArray(len.toInt()) + input.readFully(bytes) + return String(bytes, Charsets.UTF_8) + } + + private fun skipFully(input: DataInputStream, n: Long) { + var remaining = n + while (remaining > 0) { + val skipped = input.skip(remaining) + if (skipped > 0) { + remaining -= skipped + } else { + // skip() can return 0 near buffer boundaries; fall back to a read. + if (input.read() < 0) throw java.io.EOFException() + remaining -= 1 + } + } + } +} diff --git a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/LocalLlmBackend.kt b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/LocalLlmBackend.kt index f42eb393..cb9f072c 100644 --- a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/LocalLlmBackend.kt +++ b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/LocalLlmBackend.kt @@ -2,7 +2,7 @@ package com.itsaky.androidide.plugins.aicore import android.llama.cpp.LLamaAndroid import android.net.Uri -import android.provider.DocumentsContract +import android.provider.OpenableColumns import com.itsaky.androidide.plugins.services.LlmInferenceService.* import com.itsaky.androidide.plugins.services.SharedServices import com.itsaky.androidide.plugins.PluginContext @@ -49,97 +49,123 @@ class LocalLlmBackend(private val context: PluginContext) : LlmBackend { } /** - * Resolves a content URI to an actual file path that native code can access. - * If the URI points to a real file, returns the file path. Otherwise, returns null. + * Resolves the user-selected model reference to a real filesystem path the native + * loader can `fopen`. + * + * - A plain path is returned as-is. + * - A `content://` URI (what SAF `OpenDocument` returns, held with persistable read + * permission) is streamed into a private cache file and that path is returned. + * + * IMPORTANT: this loads *exactly* the file the user selected. It must never fall back + * to "some other .gguf on disk" — doing so silently loads the wrong model (e.g. an + * embedding model), which aborts native inference and takes the IDE down. See ADFA-4388. */ private fun resolveContentUriToPath(uriString: String): String? { if (!uriString.startsWith("content://")) { - return uriString // Already a file path + return uriString // Already a real file path } val uri = Uri.parse(uriString) - context.logger.info("Resolving content URI: $uri") + context.logger.info("Resolving selected model URI: $uri") - try { - // Try to get the actual file path using DocumentsContract - if (DocumentsContract.isDocumentUri(context.androidContext, uri)) { - val docId = DocumentsContract.getDocumentId(uri) - context.logger.debug("Document ID: $docId") - - // For downloads provider, the path is typically in the downloads folder - if (uri.authority == "com.android.providers.downloads.documents") { - // The docId for downloads provider can be in different formats - // Try to construct the file path - val downloadsDir = android.os.Environment.getExternalStoragePublicDirectory( - android.os.Environment.DIRECTORY_DOWNLOADS - ) - - // Try common patterns for file names in downloads - // The docId might be a number (media store ID) or "msf:number" or "raw:/path" - val filePath = when { - docId.startsWith("raw:") -> { - docId.substring(4) // Remove "raw:" prefix - } - docId.startsWith("msf:") -> { - // This is a media store file ID, we need to query it - // For now, try to find .gguf files in downloads - findGgufFileInDownloads(downloadsDir) - } - else -> { - // Might be a direct file ID, try downloads folder - findGgufFileInDownloads(downloadsDir) - } - } + val resolver = context.androidContext.contentResolver - if (filePath != null && File(filePath).exists()) { - context.logger.info("Resolved to file path: $filePath") - return filePath + // Read the selected document's display name + size (used to key the cache copy). + var displayName = "model.gguf" + var size = -1L + try { + resolver.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME, OpenableColumns.SIZE), null, null, null) + ?.use { c -> + if (c.moveToFirst()) { + val nameIdx = c.getColumnIndex(OpenableColumns.DISPLAY_NAME) + val sizeIdx = c.getColumnIndex(OpenableColumns.SIZE) + if (nameIdx >= 0 && !c.isNull(nameIdx)) displayName = c.getString(nameIdx) + if (sizeIdx >= 0 && !c.isNull(sizeIdx)) size = c.getLong(sizeIdx) } } - } } catch (e: Exception) { - context.logger.error("Error resolving content URI", e) + context.logger.warn("Could not query model metadata for $uri: ${e.message}") } - return null - } - - /** - * Scans the downloads directory for .gguf files and returns the first one found. - * This is a fallback when we can't directly resolve the content URI. - */ - private fun findGgufFileInDownloads(downloadsDir: File): String? { - if (!downloadsDir.exists() || !downloadsDir.isDirectory) { - context.logger.warn("Downloads directory not found: ${downloadsDir.absolutePath}") - return null + // Deterministic cache path keyed by URI + size, so the same selection reuses the + // same copy and a different selection can never collide with it. + val modelsDir = File(context.androidContext.filesDir, "llm-models").apply { mkdirs() } + val safeName = displayName.replace(Regex("[^A-Za-z0-9._-]"), "_") + val cacheFile = File(modelsDir, "${kotlin.math.abs(uriString.hashCode())}_${size}_$safeName") + + // Reuse a complete prior copy. + if (cacheFile.exists() && (size < 0 || cacheFile.length() == size)) { + context.logger.info("Using cached model copy: ${cacheFile.absolutePath}") + pruneOtherModels(modelsDir, cacheFile) + return cacheFile.absolutePath } - val ggufFiles = downloadsDir.listFiles { file -> - file.isFile && file.name.endsWith(".gguf", ignoreCase = true) + // Materialize the selected URI into the cache. Copy to a temp file then rename, so an + // interrupted copy can't be mistaken for a complete model on the next launch. + return try { + context.logger.info("Copying selected model into app storage: $displayName ($size bytes)") + val tmp = File(modelsDir, cacheFile.name + ".tmp") + val copied = resolver.openInputStream(uri)?.use { input -> + FileOutputStream(tmp).use { output -> input.copyTo(output, 1 shl 20) } + } + if (copied == null) { + context.logger.error("Could not open input stream for selected model $uri") + tmp.delete() + return null + } + if (size >= 0 && tmp.length() != size) { + context.logger.error("Model copy incomplete: expected $size bytes, got ${tmp.length()}") + tmp.delete() + return null + } + if (!tmp.renameTo(cacheFile)) { + tmp.copyTo(cacheFile, overwrite = true) + tmp.delete() + } + pruneOtherModels(modelsDir, cacheFile) + context.logger.info("Model ready at ${cacheFile.absolutePath}") + cacheFile.absolutePath + } catch (e: Exception) { + context.logger.error("Failed to copy selected model into app storage", e) + null } + } - if (ggufFiles != null && ggufFiles.isNotEmpty()) { - // Return the most recently modified .gguf file - val latestFile = ggufFiles.maxByOrNull { it.lastModified() } - context.logger.info("Found .gguf file in downloads: ${latestFile?.absolutePath}") - return latestFile?.absolutePath + /** + * Keeps only the active model copy in the cache dir. Model files are large, and we only + * ever need the currently-selected one on disk. Deleting a file that native code has + * already mmap'd is safe on Android — the mapping stays valid until the model is freed. + */ + private fun pruneOtherModels(modelsDir: File, keep: File) { + modelsDir.listFiles()?.forEach { f -> + if (f.absolutePath != keep.absolutePath && f.delete()) { + context.logger.debug("Pruned old model copy: ${f.name}") + } } - - context.logger.warn("No .gguf files found in downloads directory") - return null } private suspend fun ensureModelLoaded(modelPath: String) { // Resolve content URI to actual file path val resolvedPath = resolveContentUriToPath(modelPath) if (resolvedPath == null) { - throw IllegalStateException("Could not resolve model path: $modelPath. Make sure the .gguf file is in the Downloads folder.") + throw ModelNotConfiguredException("Could not read the selected model file. Re-select the .gguf model in AI Settings.") } if (modelLoaded && currentModelPath == resolvedPath) { return // Already loaded } + // Guard the chat path against encoder-only embedding models. Running causal generation on + // one aborts natively (SIGABRT) and takes the IDE down. Classify BEFORE unloading any + // working chat model, so a wrong selection never tears down a good one. See ADFA-4388. + val kind = GgufModelInspector.classify(resolvedPath) + if (kind.isEmbeddingOnly) { + throw IncompatibleModelException( + "The selected model is an embedding model and can't be used for chat. " + + "Choose a chat model in AI Settings." + ) + } + // Unload old model if loaded if (modelLoaded) { context.logger.info("Unloading previous model: $currentModelPath") @@ -220,6 +246,9 @@ class LocalLlmBackend(private val context: PluginContext) : LlmBackend { future.complete(LlmResponse.success(responseText, tokenCount, System.currentTimeMillis() - startTime)) } catch (e: Exception) { context.logger.error("Error during generation", e) + if (e is ModelNotConfiguredException || e is IncompatibleModelException) { + UserFeedback.notify(context.androidContext, e.message ?: "Local LLM is not configured.") + } future.complete(LlmResponse.failure("Error: ${e.message}")) } } @@ -283,6 +312,9 @@ class LocalLlmBackend(private val context: PluginContext) : LlmBackend { callback.onComplete(LlmResponse.success(responseBuilder.toString(), tokenCount, System.currentTimeMillis() - startTime)) } catch (e: Exception) { context.logger.error("Error during streaming generation", e) + if (e is ModelNotConfiguredException || e is IncompatibleModelException) { + UserFeedback.notify(context.androidContext, e.message ?: "Local LLM is not configured.") + } callback.onError("Error: ${e.message}") } } @@ -333,17 +365,21 @@ class LocalLlmBackend(private val context: PluginContext) : LlmBackend { * Release all resources. Called from AiCorePlugin.dispose(), which may run on * the main thread; llama.unload() drains a single-threaded native run loop and * can block while inference is in flight, so it must never run via runBlocking - * on Main. Cancel generation, then unload on a background thread. + * on Main. Cancel generation, then unload on a background thread, then stop + * the Llm-RunLoop thread so it doesn't outlive the plugin. */ fun close() { scope.cancel() - if (modelLoaded) { - CoroutineScope(Dispatchers.IO).launch { - try { - unloadModelInternal() - } catch (e: Exception) { - context.logger.error("Error unloading model during close()", e) - } + CoroutineScope(Dispatchers.IO).launch { + try { + unloadModelInternal() + } catch (e: Exception) { + context.logger.error("Error unloading model during close()", e) + } finally { + // The Llm-RunLoop executor thread exists even if no model was + // ever loaded; shut it down unconditionally so the plugin's + // classloader can be collected after unload. + llama.shutdown() } } } diff --git a/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/UserFeedback.kt b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/UserFeedback.kt new file mode 100644 index 00000000..078537b6 --- /dev/null +++ b/ai-core/src/main/kotlin/com/itsaky/androidide/plugins/aicore/UserFeedback.kt @@ -0,0 +1,62 @@ +package com.itsaky.androidide.plugins.aicore + +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.widget.Toast + +/** + * Surfaces actionable LLM errors to the user as a Toast, from anywhere in the process. + * + * ai-core is the shared LLM provider: chat, code-suggestions and code-review all funnel their + * generation through it. When something is misconfigured (e.g. no model selected), each consumer + * would otherwise fail silently in its own way — code-suggestions in particular retries on every + * keystroke, so the same error can arrive many times per second. This helper is the single place + * that turns those into one visible, throttled message for the user. + * + * Throttling is keyed by message text: an identical message shown within [COOLDOWN_MS] is + * suppressed, so typing doesn't stack a wall of toasts while still re-notifying later if the + * problem persists after the user has had a chance to act. + */ +object UserFeedback { + + private const val COOLDOWN_MS = 10_000L + + private val mainHandler = Handler(Looper.getMainLooper()) + + @Volatile private var lastMessage: String? = null + @Volatile private var lastShownAt = 0L + + /** + * Shows [message] as a long Toast, unless the identical message was already shown within the + * cooldown window. Safe to call from any thread. + */ + fun notify(context: Context, message: String) { + val now = System.currentTimeMillis() + synchronized(this) { + if (message == lastMessage && now - lastShownAt < COOLDOWN_MS) return + lastMessage = message + lastShownAt = now + } + val appContext = context.applicationContext + mainHandler.post { + Toast.makeText(appContext, message, Toast.LENGTH_LONG).show() + } + } +} + +/** + * Thrown when generation can't proceed because the local LLM isn't set up (no model selected, or + * the configured model path can't be resolved). Distinct from a runtime generation failure so the + * backend can surface it to the user instead of failing silently. The [message] is written to be + * shown directly to the user. + */ +class ModelNotConfiguredException(message: String) : IllegalStateException(message) + +/** + * Thrown when the selected local model is the wrong *kind* for the requested operation — most + * commonly an embedding (encoder-only) model selected for chat, which would abort native + * inference. Like [ModelNotConfiguredException] the [message] is written for direct display, and + * the backend surfaces it to the user instead of attempting generation. See ADFA-4388. + */ +class IncompatibleModelException(message: String) : IllegalStateException(message) diff --git a/code-suggestions-plugin/README.md b/code-suggestions-plugin/README.md new file mode 100644 index 00000000..8a3e7fc5 --- /dev/null +++ b/code-suggestions-plugin/README.md @@ -0,0 +1,73 @@ +# Code Suggestions plugin for CodeOnTheGo + +Inline **ghost-text** code completions. As you type, the plugin debounces, asks +an LLM for a completion at the cursor, and shows it as dimmed inline text via +`IdeEditorService.showInlineSuggestion()`. + +> This plugin has **no compile-time dependency** on `ai-core`; it resolves the +> inference service at runtime through the CodeOnTheGo plugin manager +> (SharedServices). Install **`ai-core` first**, then this plugin, and configure +> a model in **AI Assistant → AI Settings**. + +## Architecture + +``` +┌──────────────────────────┐ +│ code-suggestions (this) │ ← content-change listener, debounce, ghost text +└────────────┬─────────────┘ + │ SharedServices (runtime) → LlmInferenceService + ▼ +┌──────────────────────────┐ +│ ai-core │ ← LLM inference (local llama.cpp / Gemini) +└──────────────────────────┘ +``` + +## Features + +- Inline ghost-text completions while typing +- 800 ms debounce to reduce LLM load +- LRU cache to avoid redundant calls +- Language-aware prompts (Kotlin, Java, Python, …) +- Graceful degradation when `ai-core` isn't loaded yet (binds lazily) + +## Permissions + +Declared in `plugin.permissions`: + +| Permission | Why | +|---|---| +| `ide.settings` | read the configured backend/model | +| `network.access` | reach the Gemini backend when that backend is selected | + +The surrounding file content and cursor context are sent to the configured +backend to generate a completion — on-device for **Local**, or to Google over +HTTPS for **Gemini**. Choose the backend accordingly for sensitive code. + +## Building + +Prerequisites: Android SDK (API 33+), JDK 17. Create `local.properties` with +`sdk.dir=...`. No NDK or native toolchain. + +```bash +cd code-suggestions-plugin +../gradlew assemblePlugin # release -> build/plugin/code-suggestions-plugin.cgp +../gradlew assemblePluginDebug # debug variant +``` + +The build resolves `plugin-api.jar` from the repo-root `../libs/`. + +## Installation + +1. Build and install **`ai-core` first** (see [`../ai-core/README.md`](../ai-core/README.md)). +2. Build this plugin, install `build/plugin/code-suggestions-plugin.cgp` via + CodeOnTheGo's Plugin Manager, and restart the IDE. +3. Configure a model in **AI Assistant → AI Settings**. + +## Key classes + +- `CodeSuggestionsPlugin.kt` — lifecycle, content-change listener, debounce +- `SuggestionProvider.kt` — prompt building, LLM call, LRU cache + +## License + +GPL-3.0 — same as AndroidIDE / CodeOnTheGo. diff --git a/code-suggestions-plugin/build.gradle.kts b/code-suggestions-plugin/build.gradle.kts new file mode 100644 index 00000000..af56914f --- /dev/null +++ b/code-suggestions-plugin/build.gradle.kts @@ -0,0 +1,56 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("com.itsaky.androidide.plugins.build") +} + +pluginBuilder { + pluginName = "code-suggestions-plugin" +} + +android { + namespace = "com.itsaky.androidide.plugins.codesuggestions" + compileSdk = 34 + + defaultConfig { + applicationId = "com.itsaky.androidide.plugins.codesuggestions" + minSdk = 33 + targetSdk = 34 + versionCode = 1 + versionName = "1.0.0" + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + compileOnly(files("../libs/plugin-api.jar")) + + // Coroutines for async LLM calls and debouncing + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3") + + // Material Design + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.10.0") + implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.6.2") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.2") +} + +tasks.matching { + it.name.contains("checkDebugAarMetadata") || it.name.contains("checkReleaseAarMetadata") +}.configureEach { enabled = false } diff --git a/code-suggestions-plugin/gradle.properties b/code-suggestions-plugin/gradle.properties new file mode 100644 index 00000000..2c9f5454 --- /dev/null +++ b/code-suggestions-plugin/gradle.properties @@ -0,0 +1,3 @@ +android.useAndroidX=true +android.nonTransitiveRClass=true +kotlin.code.style=official diff --git a/code-suggestions-plugin/gradle/wrapper/gradle-wrapper.jar b/code-suggestions-plugin/gradle/wrapper/gradle-wrapper.jar new file mode 100755 index 00000000..8bdaf60c Binary files /dev/null and b/code-suggestions-plugin/gradle/wrapper/gradle-wrapper.jar differ diff --git a/code-suggestions-plugin/gradle/wrapper/gradle-wrapper.properties b/code-suggestions-plugin/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..d4081da4 --- /dev/null +++ b/code-suggestions-plugin/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/code-suggestions-plugin/gradlew b/code-suggestions-plugin/gradlew new file mode 100755 index 00000000..ef07e016 --- /dev/null +++ b/code-suggestions-plugin/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH="\\\"\\\"" + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/code-suggestions-plugin/gradlew.bat b/code-suggestions-plugin/gradlew.bat new file mode 100755 index 00000000..db3a6ac2 --- /dev/null +++ b/code-suggestions-plugin/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH= + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/code-suggestions-plugin/settings.gradle.kts b/code-suggestions-plugin/settings.gradle.kts new file mode 100644 index 00000000..e5798327 --- /dev/null +++ b/code-suggestions-plugin/settings.gradle.kts @@ -0,0 +1,20 @@ +pluginManagement { + repositories { google(); mavenCentral(); gradlePluginPortal() } +} + +buildscript { + repositories { google(); mavenCentral() } + dependencies { + classpath(files("../libs/plugin-api.jar")) + classpath(files("../libs/gradle-plugin.jar")) + classpath("com.android.tools.build:gradle:8.11.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.3.0") + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { google(); mavenCentral() } +} + +rootProject.name = "code-suggestions-plugin" diff --git a/code-suggestions-plugin/src/main/AndroidManifest.xml b/code-suggestions-plugin/src/main/AndroidManifest.xml new file mode 100644 index 00000000..2bb74585 --- /dev/null +++ b/code-suggestions-plugin/src/main/AndroidManifest.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/code-suggestions-plugin/src/main/assets/docs/index.html b/code-suggestions-plugin/src/main/assets/docs/index.html new file mode 100644 index 00000000..65e26548 --- /dev/null +++ b/code-suggestions-plugin/src/main/assets/docs/index.html @@ -0,0 +1,72 @@ + + + + + +Code Suggestions — Guide + + + +

Code Suggestions — Guide

+ +

Code Suggestions offers inline ghost-text completions as you + type. Inference is served by the companion AI Core plugin, so install + AI Core first and configure a model in AI Assistant → AI + Settings.

+ +

How it works

+
    +
  • As you type, the plugin waits for a short pause (debounce) and then asks + the AI Core model for a completion at the cursor.
    • +
  • The suggestion appears as dimmed ghost text after the cursor.
    • +
  • Recent suggestions are cached (LRU) so repeated contexts don't re-query + the model.
    • +
  • If AI Core isn't loaded yet, suggestions stay silent and enable + automatically once it activates.
    • +
+ +

Privacy

+
+ To generate a completion, the surrounding file content and cursor context + are sent to the configured backend. With the Local backend this stays + on-device; with the Gemini backend it is sent to Google over HTTPS. + Choose the backend accordingly for sensitive code. +
+ +

Troubleshooting

+
    +
  • No suggestions appear — confirm AI Core is installed and a model + is configured in AI Assistant's settings, then restart the IDE.
    • +
  • Suggestions feel slow — the debounce plus model latency add a + short delay after you stop typing; local models are bounded by device + speed.
    • +
+ + diff --git a/code-suggestions-plugin/src/main/assets/icon_day.png b/code-suggestions-plugin/src/main/assets/icon_day.png new file mode 100644 index 00000000..ec379a2f Binary files /dev/null and b/code-suggestions-plugin/src/main/assets/icon_day.png differ diff --git a/code-suggestions-plugin/src/main/assets/icon_night.png b/code-suggestions-plugin/src/main/assets/icon_night.png new file mode 100644 index 00000000..776fcd01 Binary files /dev/null and b/code-suggestions-plugin/src/main/assets/icon_night.png differ diff --git a/code-suggestions-plugin/src/main/kotlin/com/itsaky/androidide/plugins/codesuggestions/CodeSuggestionsPlugin.kt b/code-suggestions-plugin/src/main/kotlin/com/itsaky/androidide/plugins/codesuggestions/CodeSuggestionsPlugin.kt new file mode 100644 index 00000000..4c067732 --- /dev/null +++ b/code-suggestions-plugin/src/main/kotlin/com/itsaky/androidide/plugins/codesuggestions/CodeSuggestionsPlugin.kt @@ -0,0 +1,224 @@ +package com.itsaky.androidide.plugins.codesuggestions + +import android.util.Log +import com.itsaky.androidide.plugins.IPlugin +import com.itsaky.androidide.plugins.PluginContext +import com.itsaky.androidide.plugins.extensions.DocumentationExtension +import com.itsaky.androidide.plugins.extensions.PluginTooltipButton +import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry +import com.itsaky.androidide.plugins.services.EditorContentChangeListener +import com.itsaky.androidide.plugins.services.IdeEditorService +import com.itsaky.androidide.plugins.services.LlmInferenceService +import com.itsaky.androidide.plugins.services.SharedServices +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch + +private const val TAG = "CodeSuggestionsPlugin" +private const val DEBOUNCE_MS = 800L + +/** + * Code Suggestions Plugin provides inline ghost-text completions. + * + * Listens to editor content changes, debounces 800ms, calls LLM for suggestions, + * and displays ghost text via IdeEditorService.showInlineSuggestion(). + * + * Features: + * - LRU cache to avoid redundant LLM calls + * - Debouncing to reduce LLM load while typing + * - Language-aware suggestions (Kotlin, Java, Python, etc.) + * - Graceful degradation when ai-core plugin not loaded + */ +class CodeSuggestionsPlugin : IPlugin, DocumentationExtension { + + private val scope = CoroutineScope(Dispatchers.IO) + private lateinit var context: PluginContext + private var editorService: IdeEditorService? = null + private var suggestionProvider: SuggestionProvider? = null + + private var debounceJob: Job? = null + + private val contentChangeListener = EditorContentChangeListener { fileContent, cursorLine, cursorColumn, language -> + onEditorContentChanged(fileContent, cursorLine, cursorColumn, language) + } + + override fun initialize(context: PluginContext): Boolean { + this.context = context + Log.i(TAG, "CodeSuggestionsPlugin initialized") + return true + } + + override fun activate(): Boolean { + Log.i(TAG, "CodeSuggestionsPlugin activating...") + + // The editor service is provided by the IDE core and must be present. + editorService = context.services.get(IdeEditorService::class.java) + if (editorService == null) { + Log.w(TAG, "IdeEditorService not available - cannot display suggestions") + return false + } + + // The LLM service comes from the ai-core plugin, which may activate after + // us. Don't fail activation if it isn't ready yet - resolve it lazily on + // the first content change instead. + tryResolveLlmService() + + // Register for content change events + try { + editorService!!.addContentChangeListener(contentChangeListener) + } catch (e: Exception) { + Log.e(TAG, "Failed to register content change listener", e) + return false + } + + Log.i(TAG, "CodeSuggestionsPlugin activated") + return true + } + + /** Lazily resolves the LLM service (from ai-core) and builds the provider. */ + private fun tryResolveLlmService(): Boolean { + if (suggestionProvider != null) return true + val svc = try { + // ai-core publishes this service through SharedServices. Keep the + // context registry fallback for IDE builds that bridge shared + // services into PluginContext.services. + SharedServices.get(LlmInferenceService::class.java) + ?: context.services.get(LlmInferenceService::class.java) + } catch (e: Exception) { + Log.w(TAG, "Error resolving LlmInferenceService", e) + null + } + if (svc != null) { + suggestionProvider = SuggestionProvider(svc) + Log.i(TAG, "LlmInferenceService resolved - suggestions enabled") + return true + } + Log.w(TAG, "LlmInferenceService not available yet - install/activate the AI Core plugin") + return false + } + + override fun deactivate(): Boolean { + Log.i(TAG, "CodeSuggestionsPlugin deactivating") + debounceJob?.cancel() + try { + editorService?.removeContentChangeListener(contentChangeListener) + } catch (e: Exception) { + Log.e(TAG, "Error removing content listener", e) + } + return true + } + + override fun dispose() { + Log.i(TAG, "CodeSuggestionsPlugin disposed") + // Cancel any in-flight debounce/suggestion job and tear down the scope so + // the plugin doesn't leak coroutines after unload. + scope.cancel() + suggestionProvider?.clearCache() + } + + private fun onEditorContentChanged( + fileContent: String, + cursorLine: Int, + cursorColumn: Int, + language: String, + ) { + if (editorService == null) return + // ai-core may have activated after us; try to bind the LLM service now. + if (suggestionProvider == null && !tryResolveLlmService()) return + + // Extract the word/prefix being typed (most recent word) + val prefix = extractPrefix(fileContent, cursorLine, cursorColumn) + if (prefix.isEmpty() || prefix.length < 2) { + // Too short to suggest + editorService!!.dismissInlineSuggestion() + return + } + + // Debounce: cancel previous job and start new one + debounceJob?.cancel() + debounceJob = scope.launch { + delay(DEBOUNCE_MS) + + try { + val suggestion = suggestionProvider!!.getSuggestion( + fileContent = fileContent, + cursorLine = cursorLine, + cursorColumn = cursorColumn, + language = language, + prefix = prefix + ) + + // A newer keystroke may have cancelled us while generating. Bail without + // touching the editor so we don't dismiss the suggestion the newer job shows. + ensureActive() + + if (suggestion.isNotEmpty()) { + editorService!!.showInlineSuggestion(suggestion) + Log.d(TAG, "Showing suggestion: '$suggestion'") + } else { + editorService!!.dismissInlineSuggestion() + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + Log.e(TAG, "Error generating suggestion", e) + editorService!!.dismissInlineSuggestion() + } + } + } + + private fun extractPrefix(fileContent: String, cursorLine: Int, cursorColumn: Int): String { + return try { + val lines = fileContent.split("\n") + if (cursorLine < 0 || cursorLine >= lines.size) return "" + + val currentLine = lines[cursorLine] + if (cursorColumn < 0 || cursorColumn > currentLine.length) return "" + + // Get text from line start to cursor, extract the word being typed + val beforeCursor = currentLine.substring(0, cursorColumn) + val trimmed = beforeCursor.trimStart() + // Find last word boundary (space, symbol, etc.) + val lastBoundary = trimmed.indexOfLast { !it.isLetterOrDigit() && it != '_' } + val word = if (lastBoundary == -1) trimmed else trimmed.substring(lastBoundary + 1) + word + } catch (e: Exception) { + Log.w(TAG, "Error extracting prefix", e) + "" + } + } + + override fun getTooltipCategory(): String = "plugin_code_suggestions" + + override fun getTooltipEntries(): List = listOf( + PluginTooltipEntry( + tag = TOOLTIP_TAG_PLUGIN, + summary = "Code Suggestions shows inline ghost-text completions after a short pause while typing.", + detail = """ +

Code Suggestions listens to editor changes, debounces + typing, and asks the configured AI Core backend for a completion + at the cursor.

+

The surrounding file context is sent to the selected backend: + on-device for Local, or to Google over HTTPS for Gemini.

+ """.trimIndent(), + buttons = listOf( + PluginTooltipButton( + description = "Code Suggestions guide", + uri = "index.html", + order = 0 + ) + ) + ) + ) + + override fun getTier3DocsAssetPath(): String = "docs" + + private companion object { + const val TOOLTIP_TAG_PLUGIN = "plugin_code_suggestions" + } +} diff --git a/code-suggestions-plugin/src/main/kotlin/com/itsaky/androidide/plugins/codesuggestions/SuggestionProvider.kt b/code-suggestions-plugin/src/main/kotlin/com/itsaky/androidide/plugins/codesuggestions/SuggestionProvider.kt new file mode 100644 index 00000000..a1a0f73f --- /dev/null +++ b/code-suggestions-plugin/src/main/kotlin/com/itsaky/androidide/plugins/codesuggestions/SuggestionProvider.kt @@ -0,0 +1,102 @@ +package com.itsaky.androidide.plugins.codesuggestions + +import android.util.Log +import com.itsaky.androidide.plugins.services.LlmInferenceService +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.util.LinkedHashMap + +private const val TAG = "SuggestionProvider" + +/** + * Generates code suggestions using an LLM with LRU caching. + * Avoids redundant LLM calls for identical context. + */ +class SuggestionProvider(private val llmService: LlmInferenceService) { + + private val cache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: Map.Entry?): Boolean { + return size > MAX_CACHE_SIZE + } + } + + /** + * Generates a code suggestion for the given context. + * Uses cache to avoid redundant LLM calls. + * + * @param fileContent Full content of the current file + * @param cursorLine Line number where cursor is (0-indexed) + * @param cursorColumn Column number where cursor is (0-indexed) + * @param language Programming language (kotlin, java, etc.) + * @param prefix Recently typed text (for context) + * @return Suggested completion text or empty if no suggestion + */ + suspend fun getSuggestion( + fileContent: String, + cursorLine: Int, + cursorColumn: Int, + language: String, + prefix: String, + ): String = withContext(Dispatchers.IO) { + return@withContext try { + val cacheKey = "$prefix|$language" + cache[cacheKey]?.let { + Log.d(TAG, "Cache hit for: $prefix") + return@withContext it + } + + // Build up-to-cursor context from the file, bounded to the last 500 chars. + val lines = fileContent.split("\n") + val contextBefore = buildString { + for (i in 0 until cursorLine.coerceIn(0, lines.size)) { + append(lines[i]).append("\n") + } + if (cursorLine in lines.indices) { + val line = lines[cursorLine] + append(line.substring(0, cursorColumn.coerceIn(0, line.length))) + } + }.takeLast(500) + + val prompt = """ + You are a $language code completion assistant. + Complete the following code. Return ONLY the completion text, no explanation. + + Context: + $contextBefore + + Complete this: + """.trimIndent() + + val config = LlmInferenceService.LlmConfig("local") + val response = llmService.generateCompletion(prompt, config).get() + if (response.success) { + val suggestion = response.text.trim() + .takeWhile { !it.isWhitespace() || it == ' ' } // Take tokens until newline + .trim() + + if (suggestion.isNotEmpty()) { + cache[cacheKey] = suggestion + Log.d(TAG, "Generated suggestion for '$prefix': $suggestion") + suggestion + } else { + "" + } + } else { + Log.w(TAG, "LLM error: ${response.error}") + "" + } + } catch (e: Exception) { + Log.e(TAG, "Error generating suggestion", e) + "" + } + } + + fun clearCache() { + cache.clear() + Log.d(TAG, "Cache cleared") + } + + companion object { + private const val MAX_CACHE_SIZE = 100 + } +} diff --git a/code-suggestions-plugin/src/main/res/values/colors.xml b/code-suggestions-plugin/src/main/res/values/colors.xml new file mode 100644 index 00000000..43e21263 --- /dev/null +++ b/code-suggestions-plugin/src/main/res/values/colors.xml @@ -0,0 +1,7 @@ + + + #485D92 + #FFFFFF + #DAE2FF + #001847 + diff --git a/code-suggestions-plugin/src/main/res/values/styles.xml b/code-suggestions-plugin/src/main/res/values/styles.xml new file mode 100644 index 00000000..17bb8c55 --- /dev/null +++ b/code-suggestions-plugin/src/main/res/values/styles.xml @@ -0,0 +1,9 @@ + + + + diff --git a/speech-to-text-plugin/README.md b/speech-to-text-plugin/README.md new file mode 100644 index 00000000..7f5d3ccd --- /dev/null +++ b/speech-to-text-plugin/README.md @@ -0,0 +1,83 @@ +# Speech to Text plugin for CodeOnTheGo + +Voice-to-code. Adds a **Voice to Code** button to the editor toolbar: tap it, +speak, and the recognized text — or code generated from it — is inserted at the +cursor. Speech recognition uses Android's `SpeechRecognizer` (on-device when +available); code generation is delegated to the companion `ai-core` plugin. + +> Code generation has **no compile-time dependency** on `ai-core`; the LLM +> service is resolved at runtime. Install **`ai-core` first** for voice→code. +> Without it, the raw transcript is inserted instead. + +## Architecture + +``` +┌──────────────────────────┐ +│ speech-to-text (this) │ ← toolbar button, SpeechRecognizer, insert at cursor +└────────────┬─────────────┘ + │ SharedServices (runtime) → LlmInferenceService (optional) + ▼ +┌──────────────────────────┐ +│ ai-core │ ← turns the transcript into code +└──────────────────────────┘ +``` + +## Features + +- "Voice to Code" toolbar action (`UIExtension`), enabled only while a file is open +- Dynamic icon: mic (idle) → waves (recording) → spinner (processing) +- On-device speech recognition preferred, with network fallback +- Optional LLM code generation from the transcript via `ai-core` +- Inserts the result at the editor cursor + +## Permissions + +Android (`uses-permission`): `RECORD_AUDIO`, `INTERNET`. + +Plugin (`plugin.permissions`): `filesystem.read`, `filesystem.write`, +`network.access`, `native.code`. + +| Permission | Why | +|---|---| +| `RECORD_AUDIO` | capture microphone audio for recognition | +| `INTERNET` / `network.access` | network speech fallback and the Gemini backend | +| `filesystem.write` | insert transcribed/generated text into the open file | +| `filesystem.read` | read editor/file context | + +`RECORD_AUDIO` is a runtime (dangerous) permission: it is requested on first tap +of the button, not at load. If denied, the plugin degrades gracefully (no voice +capture) and can still insert LLM output. Microphone audio is captured only +while recording and is not stored by the plugin. When `ai-core`'s **Gemini** +backend is selected, the transcript is sent to Google over HTTPS; the **Local** +backend keeps everything on-device. + +## Building + +Prerequisites: Android SDK (API 33+), JDK 17. Create `local.properties` with +`sdk.dir=...`. + +```bash +cd speech-to-text-plugin +../gradlew assemblePlugin # release -> build/plugin/speech-to-text-plugin.cgp +../gradlew assemblePluginDebug # debug variant +``` + +The build resolves `plugin-api.jar` from the repo-root `../libs/`. + +## Installation + +1. (Optional but recommended) Build and install **`ai-core` first** for + voice→code (see [`../ai-core/README.md`](../ai-core/README.md)). +2. Build this plugin, install `build/plugin/speech-to-text-plugin.cgp` via + CodeOnTheGo's Plugin Manager, and restart the IDE. +3. Open a file, tap the microphone in the editor toolbar, grant the microphone + permission on first use, and speak. + +## Key classes + +- `SpeechToTextPlugin.kt` — toolbar action, `SpeechRecognizer` lifecycle, + transcript handling, editor insertion + +## License + +GPL-3.0 — same as AndroidIDE / CodeOnTheGo. diff --git a/speech-to-text-plugin/build.gradle.kts b/speech-to-text-plugin/build.gradle.kts new file mode 100644 index 00000000..514dd4a4 --- /dev/null +++ b/speech-to-text-plugin/build.gradle.kts @@ -0,0 +1,64 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("com.itsaky.androidide.plugins.build") +} + +pluginBuilder { + pluginName = "speech-to-text-plugin" +} + +android { + namespace = "com.itsaky.androidide.plugins.stt" + compileSdk = 34 + + defaultConfig { + applicationId = "com.itsaky.androidide.plugins.stt" + minSdk = 33 + targetSdk = 34 + versionCode = 1 + versionName = "1.0.0" + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + buildFeatures { + viewBinding = true + } +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + compileOnly(files("../libs/plugin-api.jar")) + + // ONNX Runtime for offline speech recognition (Moonshine) + implementation("com.microsoft.onnxruntime:onnxruntime-android:1.17.0") + + // Coroutines + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3") + + // Material Design & AndroidX + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("androidx.core:core:1.12.0") + implementation("com.google.android.material:material:1.10.0") + implementation("androidx.lifecycle:lifecycle-viewmodel-ktx:2.6.2") + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.2") +} + +tasks.matching { + it.name.contains("checkDebugAarMetadata") || it.name.contains("checkReleaseAarMetadata") +}.configureEach { enabled = false } diff --git a/speech-to-text-plugin/gradle.properties b/speech-to-text-plugin/gradle.properties new file mode 100644 index 00000000..ddc879cb --- /dev/null +++ b/speech-to-text-plugin/gradle.properties @@ -0,0 +1,6 @@ +android.useAndroidX=true +android.nonTransitiveRClass=true +kotlin.code.style=official +# The ONNX Runtime dependency makes dex/resource merging memory-hungry; the Gradle +# defaults (512m heap / 384m metaspace) OOM during assemblePlugin. Raise them. +org.gradle.jvmargs=-Xmx2g -XX:MaxMetaspaceSize=1g diff --git a/speech-to-text-plugin/gradle/wrapper/gradle-wrapper.jar b/speech-to-text-plugin/gradle/wrapper/gradle-wrapper.jar new file mode 100755 index 00000000..8bdaf60c Binary files /dev/null and b/speech-to-text-plugin/gradle/wrapper/gradle-wrapper.jar differ diff --git a/speech-to-text-plugin/gradle/wrapper/gradle-wrapper.properties b/speech-to-text-plugin/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..d4081da4 --- /dev/null +++ b/speech-to-text-plugin/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/speech-to-text-plugin/gradlew b/speech-to-text-plugin/gradlew new file mode 100755 index 00000000..ef07e016 --- /dev/null +++ b/speech-to-text-plugin/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH="\\\"\\\"" + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/speech-to-text-plugin/gradlew.bat b/speech-to-text-plugin/gradlew.bat new file mode 100755 index 00000000..db3a6ac2 --- /dev/null +++ b/speech-to-text-plugin/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH= + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/speech-to-text-plugin/settings.gradle.kts b/speech-to-text-plugin/settings.gradle.kts new file mode 100644 index 00000000..4ebe4103 --- /dev/null +++ b/speech-to-text-plugin/settings.gradle.kts @@ -0,0 +1,20 @@ +pluginManagement { + repositories { google(); mavenCentral(); gradlePluginPortal() } +} + +buildscript { + repositories { google(); mavenCentral() } + dependencies { + classpath(files("../libs/plugin-api.jar")) + classpath(files("../libs/gradle-plugin.jar")) + classpath("com.android.tools.build:gradle:8.11.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.3.0") + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { google(); mavenCentral() } +} + +rootProject.name = "speech-to-text-plugin" diff --git a/speech-to-text-plugin/src/main/AndroidManifest.xml b/speech-to-text-plugin/src/main/AndroidManifest.xml new file mode 100644 index 00000000..6397e458 --- /dev/null +++ b/speech-to-text-plugin/src/main/AndroidManifest.xml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/speech-to-text-plugin/src/main/assets/docs/index.html b/speech-to-text-plugin/src/main/assets/docs/index.html new file mode 100644 index 00000000..1496810a --- /dev/null +++ b/speech-to-text-plugin/src/main/assets/docs/index.html @@ -0,0 +1,78 @@ + + + + + +Speech to Text — Guide + + + +

Speech to Text — Voice to Code

+ +

Speech to Text adds a Voice to Code button to the editor + toolbar (the microphone icon). Tap it, speak, and the recognized text — or + code generated from it — is inserted at the cursor.

+ +

Using it

+
    +
  • Open a file in the editor (the button is disabled when nothing is open).
    • +
  • Tap the microphone. On first use, grant the microphone permission + and tap again.
    • +
  • Speak your request. The icon shows the state: mic (idle) → waves + (recording) → spinner (processing).
    • +
  • If AI Core is installed, the transcript is turned into code by the + model; otherwise the raw transcript is inserted.
    • +
+ +

On-device vs. cloud recognition

+

The plugin prefers Android's on-device speech recognition. If the + device lacks an offline recognizer, Android may fall back to its network + recognition service.

+ +

Privacy

+
+ Speech recognition is handled by Android's recognizer (on-device when + available). When AI Core is used to generate code, the transcript is sent to + the configured backend — on-device for Local, or to Google over HTTPS + for Gemini. Audio is captured only while recording and is not stored + by the plugin. +
+ +

Troubleshooting

+
    +
  • Button greyed out — open a file first; voice input inserts at the + editor cursor.
    • +
  • "Speech recognition is not available" — the device has no speech + recognizer; install/enable Google's speech services.
    • +
  • Permission prompt didn't appear — grant Microphone under + Settings → Apps → CodeOnTheGo → Permissions, then tap again.
    • +
+ + diff --git a/speech-to-text-plugin/src/main/assets/icon_day.png b/speech-to-text-plugin/src/main/assets/icon_day.png new file mode 100644 index 00000000..c7478961 Binary files /dev/null and b/speech-to-text-plugin/src/main/assets/icon_day.png differ diff --git a/speech-to-text-plugin/src/main/assets/icon_night.png b/speech-to-text-plugin/src/main/assets/icon_night.png new file mode 100644 index 00000000..7a55d930 Binary files /dev/null and b/speech-to-text-plugin/src/main/assets/icon_night.png differ diff --git a/speech-to-text-plugin/src/main/kotlin/com/itsaky/androidide/plugins/stt/SpeechToTextPlugin.kt b/speech-to-text-plugin/src/main/kotlin/com/itsaky/androidide/plugins/stt/SpeechToTextPlugin.kt new file mode 100644 index 00000000..f88d99b1 --- /dev/null +++ b/speech-to-text-plugin/src/main/kotlin/com/itsaky/androidide/plugins/stt/SpeechToTextPlugin.kt @@ -0,0 +1,458 @@ +package com.itsaky.androidide.plugins.stt + +import android.Manifest +import android.app.Activity +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import android.util.Log +import android.widget.Toast +import androidx.core.content.ContextCompat +import com.itsaky.androidide.plugins.IPlugin +import com.itsaky.androidide.plugins.PluginContext +import com.itsaky.androidide.plugins.extensions.DocumentationExtension +import com.itsaky.androidide.plugins.extensions.PluginTooltipButton +import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry +import com.itsaky.androidide.plugins.extensions.ShowAsAction +import com.itsaky.androidide.plugins.extensions.ToolbarAction +import com.itsaky.androidide.plugins.extensions.UIExtension +import com.itsaky.androidide.plugins.services.IdeEditorService +import com.itsaky.androidide.plugins.services.IdeUIService +import com.itsaky.androidide.plugins.services.LlmInferenceService +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +private const val TAG = "SpeechToTextPlugin" + +/** + * Speech-to-Text Plugin provides voice-to-code capabilities. + * + * Features: + * - Voice capture via Android [SpeechRecognizer] (on-device when available) + * - Optional LLM-based code generation from the transcript + * - Editor integration for inserting the result at the cursor + * + * The plugin surfaces a "Voice to Code" action in the editor toolbar by + * implementing [UIExtension.getToolbarActions]. The action is only visible + * while a file is open in the editor. + */ +class SpeechToTextPlugin : IPlugin, UIExtension, DocumentationExtension { + + private val scope = CoroutineScope(Dispatchers.IO) + private lateinit var context: PluginContext + private var llmService: LlmInferenceService? = null + private var editorService: IdeEditorService? = null + private var uiService: IdeUIService? = null + + /** Held only between startListening() and the terminal result/error callback. */ + private var speechRecognizer: SpeechRecognizer? = null + + /** Drives the toolbar icon via [ToolbarAction.iconProvider]. */ + private enum class RecordingState { IDLE, RECORDING, PROCESSING } + + @Volatile + private var recordingState = RecordingState.IDLE + + /** + * Updates the recording state and asks the IDE to rebuild the toolbar so the + * icon provider is re-evaluated and the new (possibly animated) icon is shown. + */ + private fun setState(state: RecordingState) { + recordingState = state + uiService?.refreshToolbarActions() + } + + private fun currentIconRes(): Int = when (recordingState) { + RecordingState.IDLE -> R.drawable.ic_mic + RecordingState.RECORDING -> R.drawable.ic_waves + RecordingState.PROCESSING -> R.drawable.ic_processing + } + + override fun initialize(context: PluginContext): Boolean { + this.context = context + Log.i(TAG, "SpeechToTextPlugin initialized") + return true + } + + override fun activate(): Boolean { + Log.i(TAG, "SpeechToTextPlugin activating...") + + // Get services from plugin context + llmService = context.services.get(LlmInferenceService::class.java) + editorService = context.services.get(IdeEditorService::class.java) + uiService = context.services.get(IdeUIService::class.java) + + if (llmService == null) { + Log.w(TAG, "LlmInferenceService not available - voice generation disabled") + } + if (editorService == null) { + Log.w(TAG, "IdeEditorService not available - editor integration disabled") + } + if (uiService == null) { + Log.w(TAG, "IdeUIService not available - toolbar icon will not animate between states") + } + + Log.i(TAG, "SpeechToTextPlugin activated") + return true + } + + override fun deactivate(): Boolean { + Log.i(TAG, "SpeechToTextPlugin deactivating") + destroyRecognizer() + return true + } + + override fun dispose() { + Log.i(TAG, "SpeechToTextPlugin disposed") + destroyRecognizer() + // Tear down the transcript-processing scope so no LLM/generation coroutine + // outlives the plugin after unload. + scope.cancel() + } + + /** + * Contributes the "Voice to Code" button to the editor toolbar. The IDE + * registers this via [UIExtension] and shows it while an editor is open. + */ + override fun getToolbarActions(): List = listOf( + ToolbarAction( + id = TOOLBAR_ACTION_ID, + title = "Voice to Code", + // Static fallback for hosts that don't support iconProvider. + icon = R.drawable.ic_mic, + showAsAction = ShowAsAction.IF_ROOM, + order = 100, + action = { startVoiceCapture() } + ).apply { + // Dynamic icon: mic (idle) -> animated waves (recording) -> spinner (processing). + iconProvider = { currentIconRes() } + // Voice-to-code inserts into the active editor, so the button is only usable + // while a file is open. The host greys it out and blocks taps when this is false, + // and re-evaluates it whenever the toolbar is rebuilt (including editor changes). + isEnabledProvider = { hasOpenFile() } + } + ) + + /** True when there is a file open in the editor to insert transcribed text into. */ + private fun hasOpenFile(): Boolean = try { + editorService?.getCurrentFile() != null + } catch (e: Exception) { + false + } + override fun getTooltipCategory(): String = "plugin_speech_to_text" + + override fun getTooltipEntries(): List = listOf( + PluginTooltipEntry( + tag = TOOLBAR_ACTION_ID, + summary = "Voice to Code: tap, speak, and insert the transcript — or code generated from it — at the cursor.", + detail = """ +

The microphone button in the editor toolbar records a + short voice command and inserts the result at the cursor.

+

Recognition uses Android's on-device recognizer when available. + If the AI Core plugin is installed, the transcript is turned + into code by the model; otherwise the raw transcript is inserted.

+

The button is enabled only while a file is open, and microphone + permission is requested on first use.

+ """.trimIndent(), + buttons = listOf( + PluginTooltipButton( + description = "Speech to Text guide", + uri = "index.html", + order = 0 + ) + ) + ) + ) + + /** Subdirectory under src/main/assets/ holding the Tier 3 offline docs. */ + override fun getTier3DocsAssetPath(): String = "docs" + + /** + * Entry point for the toolbar action. Runs on the UI thread (the toolbar + * action item requires it), which is also required to construct and drive + * [SpeechRecognizer]. + */ + private fun startVoiceCapture() { + val ctx = hostContext() + + // Belt-and-suspenders: the toolbar already disables the button when no file is open, + // but guard here too so a stale enabled state can't start a pointless recording. + if (!hasOpenFile()) { + toast("Open a file in the editor to insert voice input.") + return + } + + if (!hasMicrophonePermission()) { + requestMicrophonePermission() + toast("Microphone permission needed. Grant it, then tap again.") + return + } + + if (!SpeechRecognizer.isRecognitionAvailable(ctx)) { + toast("Speech recognition is not available on this device.") + return + } + + try { + destroyRecognizer() + val recognizer = SpeechRecognizer.createSpeechRecognizer(ctx) + recognizer.setRecognitionListener(recognitionListener) + speechRecognizer = recognizer + + val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra( + RecognizerIntent.EXTRA_LANGUAGE_MODEL, + RecognizerIntent.LANGUAGE_MODEL_FREE_FORM + ) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, false) + // Prefer on-device recognition; falls back to network if unsupported. + putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true) + } + toast("Listening… speak your command.") + recognizer.startListening(intent) + setState(RecordingState.RECORDING) + } catch (e: Exception) { + Log.e(TAG, "Failed to start speech recognition", e) + toast("Could not start recording: ${e.message}") + destroyRecognizer() + setState(RecordingState.IDLE) + } + } + + private val recognitionListener = object : RecognitionListener { + override fun onResults(results: Bundle?) { + val transcript = results + ?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + ?.firstOrNull() + ?.trim() + destroyRecognizer() + if (transcript.isNullOrBlank()) { + toast("Didn't catch that. Try again.") + setState(RecordingState.IDLE) + return + } + setState(RecordingState.PROCESSING) + handleTranscript(transcript) + } + + override fun onError(error: Int) { + destroyRecognizer() + toast("Recording error: ${describeError(error)}") + setState(RecordingState.IDLE) + } + + override fun onReadyForSpeech(params: Bundle?) {} + override fun onBeginningOfSpeech() {} + override fun onRmsChanged(rmsdB: Float) {} + override fun onBufferReceived(buffer: ByteArray?) {} + override fun onEndOfSpeech() {} + override fun onPartialResults(partialResults: Bundle?) {} + override fun onEvent(eventType: Int, params: Bundle?) {} + } + + /** + * Takes the recognized text, optionally runs it through the LLM to produce + * code, and inserts the result at the cursor. + */ + private fun handleTranscript(transcript: String) { + Log.i(TAG, "Recognized: $transcript") + scope.launch { + val output = if (llmService != null) { + generateCodeFromVoice(transcript) + } else { + transcript + } + withContext(Dispatchers.Main) { + try { + if (output.startsWith("Error:")) { + toast(output) + return@withContext + } + val inserted = insertCodeAtCursor(output) + toast( + if (inserted) "Inserted from voice." + else "Recognized: \"$transcript\" (open a file to insert)." + ) + } finally { + // Back to idle (mic) regardless of how processing ended. + setState(RecordingState.IDLE) + } + } + } + } + + /** + * Generates code from a voice command using the LLM. + * + * @param voiceText The transcribed text from speech-to-text + * @param language Programming language context (e.g., "kotlin", "java") + * @return Generated code snippet or error message + */ + suspend fun generateCodeFromVoice(voiceText: String, language: String = "kotlin"): String { + return try { + if (llmService == null) { + return "Error: LlmInferenceService not available" + } + + // Build a completion prompt for code generation + val prompt = """ + User request: $voiceText + + Generate $language code to fulfill this request. Return only the code, no explanation. + Code: + """.trimIndent() + + val config = LlmInferenceService.LlmConfig("local") + val response = llmService!!.generateCompletion(prompt, config).get() + if (response.success) { + response.text.trim() + } else { + "Error: ${response.error}" + } + } catch (e: Exception) { + Log.e(TAG, "Error generating code from voice", e) + "Error: ${e.message}" + } + } + + /** + * Inserts generated code at the cursor position. + */ + fun insertCodeAtCursor(code: String): Boolean { + // insertTextAtCursor throws SecurityException if the plugin lacks FILESYSTEM_WRITE + // (declared in the manifest). Never let a service error crash the host process. + return try { + editorService?.insertTextAtCursor(code) ?: false + } catch (e: Exception) { + Log.e(TAG, "Failed to insert text at cursor", e) + false + } + } + + /** + * Recognizes voice commands (REFACTOR, CREATE_CLASS, etc.). + * Returns the intent type or null if no match. + */ + fun recognizeIntent(voiceText: String): String? { + val text = voiceText.lowercase() + return when { + text.contains("refactor") -> "REFACTOR" + text.contains("create") && (text.contains("class") || text.contains("function")) -> "CREATE_CLASS" + text.contains("undo") -> "UNDO" + text.contains("redo") -> "REDO" + text.contains("format") -> "FORMAT" + text.contains("delete") || text.contains("remove") -> "DELETE" + else -> null + } + } + + /** + * Checks if microphone permission is granted. + * @return true if RECORD_AUDIO permission is granted, false otherwise + */ + fun hasMicrophonePermission(): Boolean { + return ContextCompat.checkSelfPermission( + hostContext(), + Manifest.permission.RECORD_AUDIO + ) == PackageManager.PERMISSION_GRANTED + } + + /** + * Requests microphone permission from the user. + * Returns immediately if already granted. + */ + fun requestMicrophonePermission() { + if (hasMicrophonePermission()) { + Log.d(TAG, "Microphone permission already granted") + return + } + + Log.d(TAG, "Requesting microphone permission...") + try { + // Must be the host Activity: the plugin's androidContext is a ContextThemeWrapper, + // never an Activity, and RECORD_AUDIO is owned by the host app's UID. + val activity = hostActivity() + if (activity != null && android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.M) { + activity.requestPermissions( + arrayOf(Manifest.permission.RECORD_AUDIO), + PERMISSION_REQUEST_CODE + ) + } else { + Log.w(TAG, "No host Activity available to request microphone permission") + } + } catch (e: Exception) { + Log.e(TAG, "Error requesting microphone permission", e) + } + } + + private fun destroyRecognizer() { + val recognizer = speechRecognizer ?: return + speechRecognizer = null + runOnMain { + try { + recognizer.destroy() + } catch (e: Exception) { + Log.w(TAG, "Failed to destroy SpeechRecognizer", e) + } + } + } + + private fun describeError(error: Int): String = when (error) { + SpeechRecognizer.ERROR_AUDIO -> "audio recording error" + SpeechRecognizer.ERROR_CLIENT -> "client error" + SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS -> "insufficient permissions" + SpeechRecognizer.ERROR_NETWORK -> "network error" + SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "network timeout" + SpeechRecognizer.ERROR_NO_MATCH -> "no speech matched" + SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "recognizer busy" + SpeechRecognizer.ERROR_SERVER -> "server error" + SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "no speech detected" + else -> "code $error" + } + + /** + * The foreground host Activity, if any. Required for anything that talks to the + * window manager or requests runtime permissions — the plugin's own + * [PluginContext.androidContext] reports the plugin package while running under the + * host UID, which the framework rejects (SecurityException: package not in UID). + */ + private fun hostActivity(): Activity? = uiService?.getCurrentActivity() + + /** + * A host-owned Context (package + UID match the running process). Prefer the + * foreground Activity; fall back to the host application context, which the + * plugin resource context delegates to. + */ + private fun hostContext(): Context = + hostActivity() ?: context.androidContext.applicationContext + + private fun toast(message: String) = runOnMain { + try { + Toast.makeText(hostContext(), message, Toast.LENGTH_SHORT).show() + } catch (e: Exception) { + Log.w(TAG, "Failed to show toast", e) + } + } + + private fun runOnMain(block: () -> Unit) { + if (Looper.myLooper() == Looper.getMainLooper()) { + block() + } else { + Handler(Looper.getMainLooper()).post(block) + } + } + + companion object { + private const val PERMISSION_REQUEST_CODE = 100 + private const val TOOLBAR_ACTION_ID = "stt_voice_to_code" + } +} diff --git a/speech-to-text-plugin/src/main/res/drawable/ic_mic.xml b/speech-to-text-plugin/src/main/res/drawable/ic_mic.xml new file mode 100644 index 00000000..cb0245cc --- /dev/null +++ b/speech-to-text-plugin/src/main/res/drawable/ic_mic.xml @@ -0,0 +1,14 @@ + + + + + + diff --git a/speech-to-text-plugin/src/main/res/drawable/ic_processing.xml b/speech-to-text-plugin/src/main/res/drawable/ic_processing.xml new file mode 100644 index 00000000..f609dfe1 --- /dev/null +++ b/speech-to-text-plugin/src/main/res/drawable/ic_processing.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + diff --git a/speech-to-text-plugin/src/main/res/drawable/ic_waves.xml b/speech-to-text-plugin/src/main/res/drawable/ic_waves.xml new file mode 100644 index 00000000..f9e3b665 --- /dev/null +++ b/speech-to-text-plugin/src/main/res/drawable/ic_waves.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/speech-to-text-plugin/src/main/res/values/colors.xml b/speech-to-text-plugin/src/main/res/values/colors.xml new file mode 100644 index 00000000..43e21263 --- /dev/null +++ b/speech-to-text-plugin/src/main/res/values/colors.xml @@ -0,0 +1,7 @@ + + + #485D92 + #FFFFFF + #DAE2FF + #001847 + diff --git a/speech-to-text-plugin/src/main/res/values/styles.xml b/speech-to-text-plugin/src/main/res/values/styles.xml new file mode 100644 index 00000000..17bb8c55 --- /dev/null +++ b/speech-to-text-plugin/src/main/res/values/styles.xml @@ -0,0 +1,9 @@ + + + + diff --git a/vector-search-plugin/README.md b/vector-search-plugin/README.md new file mode 100644 index 00000000..6e9780ad --- /dev/null +++ b/vector-search-plugin/README.md @@ -0,0 +1,78 @@ +# Vector Search plugin for CodeOnTheGo + +Semantic (meaning-based) code search. Files are chunked and embedded into +vectors; a query is embedded the same way and ranked by cosine similarity. The +plugin contributes a **"Semantic Results"** section to the project search screen +via `ProjectSearchExtension`. + +> Embeddings come from the `ai-core` plugin at runtime (no compile-time +> dependency). Install **`ai-core` first** for real model embeddings. Without +> it, a lightweight **lexical** embedding fallback keeps search working at lower +> quality. + +## Architecture + +``` +┌──────────────────────────┐ +│ vector-search (this) │ ← chunk, embed, cosine-similarity ranking, project search +└────────────┬─────────────┘ + │ SharedServices (runtime) → LlmInferenceService (embeddings) + ▼ +┌──────────────────────────┐ +│ ai-core │ ← embedding generation (falls back to lexical if absent) +└──────────────────────────┘ +``` + +## Features + +- Semantic search over the current project via `ProjectSearchExtension` +- On-device embeddings through `ai-core`, with a lexical fallback +- Chunk-level results with file, line range, and a preview snippet +- Local SQLite embedding store; on-demand indexing per searched root + +## Permissions + +Declared in `plugin.permissions`: + +| Permission | Why | +|---|---| +| `filesystem.read` | read project files to chunk and embed | +| `project.structure` | enumerate the project's source roots | + +Indexing reads files in the current project only. Embeddings are stored in a +local database on the device. If `ai-core`'s **Gemini** backend is selected for +embeddings, chunk text is sent to Google over HTTPS; the **Local** backend and +the lexical fallback keep everything on-device. + +## Building + +Prerequisites: Android SDK (API 33+), JDK 17. Create `local.properties` with +`sdk.dir=...`. No NDK or native toolchain. + +```bash +cd vector-search-plugin +../gradlew assemblePlugin # release -> build/plugin/vector-search-plugin.cgp +../gradlew assemblePluginDebug # debug variant +``` + +The build resolves `plugin-api.jar` from the repo-root `../libs/`. + +## Installation + +1. Build and install **`ai-core` first** for quality embeddings (see + [`../ai-core/README.md`](../ai-core/README.md)). +2. Build this plugin, install `build/plugin/vector-search-plugin.cgp` via + CodeOnTheGo's Plugin Manager, and restart the IDE. +3. Run a query from the project search screen; look for the **Semantic + Results** section. + +## Key classes + +- `VectorSearchPlugin.kt` — lifecycle, `ProjectSearchExtension`, search flow +- `EmbeddingIndexingService.kt` — file collection, embedding storage (SQLite) +- `CodeChunker.kt` — splits files into embeddable chunks +- `VectorSearchService.kt` / `VectorMath.kt` — similarity ranking + +## License + +GPL-3.0 — same as AndroidIDE / CodeOnTheGo. diff --git a/vector-search-plugin/build.gradle.kts b/vector-search-plugin/build.gradle.kts new file mode 100644 index 00000000..492c58b4 --- /dev/null +++ b/vector-search-plugin/build.gradle.kts @@ -0,0 +1,61 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("com.itsaky.androidide.plugins.build") +} + +pluginBuilder { + pluginName = "vector-search-plugin" +} + +android { + namespace = "com.itsaky.androidide.plugins.vectorsearch" + compileSdk = 34 + + defaultConfig { + applicationId = "com.itsaky.androidide.plugins.vectorsearch" + minSdk = 33 + targetSdk = 34 + versionCode = 1 + versionName = "1.0.0" + } + + buildTypes { + release { + isMinifyEnabled = false + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + buildFeatures { + viewBinding = false + } +} + +kotlin { + compilerOptions { + jvmTarget.set(org.jetbrains.kotlin.gradle.dsl.JvmTarget.JVM_17) + } +} + +dependencies { + compileOnly(files("../libs/plugin-api.jar")) + + // Coroutines for async operations + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3") + + // SQLite for embeddings storage (Android built-in, but explicit for clarity) + implementation("androidx.sqlite:sqlite:2.4.0") + + // Material Design + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.10.0") +} + +tasks.matching { + it.name.contains("checkDebugAarMetadata") || it.name.contains("checkReleaseAarMetadata") +}.configureEach { enabled = false } diff --git a/vector-search-plugin/gradle.properties b/vector-search-plugin/gradle.properties new file mode 100644 index 00000000..2c9f5454 --- /dev/null +++ b/vector-search-plugin/gradle.properties @@ -0,0 +1,3 @@ +android.useAndroidX=true +android.nonTransitiveRClass=true +kotlin.code.style=official diff --git a/vector-search-plugin/gradle/wrapper/gradle-wrapper.jar b/vector-search-plugin/gradle/wrapper/gradle-wrapper.jar new file mode 100755 index 00000000..8bdaf60c Binary files /dev/null and b/vector-search-plugin/gradle/wrapper/gradle-wrapper.jar differ diff --git a/vector-search-plugin/gradle/wrapper/gradle-wrapper.properties b/vector-search-plugin/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..d4081da4 --- /dev/null +++ b/vector-search-plugin/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/vector-search-plugin/gradlew b/vector-search-plugin/gradlew new file mode 100755 index 00000000..ef07e016 --- /dev/null +++ b/vector-search-plugin/gradlew @@ -0,0 +1,251 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH="\\\"\\\"" + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/vector-search-plugin/gradlew.bat b/vector-search-plugin/gradlew.bat new file mode 100755 index 00000000..db3a6ac2 --- /dev/null +++ b/vector-search-plugin/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH= + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/vector-search-plugin/settings.gradle.kts b/vector-search-plugin/settings.gradle.kts new file mode 100644 index 00000000..79f610a6 --- /dev/null +++ b/vector-search-plugin/settings.gradle.kts @@ -0,0 +1,20 @@ +pluginManagement { + repositories { google(); mavenCentral(); gradlePluginPortal() } +} + +buildscript { + repositories { google(); mavenCentral() } + dependencies { + classpath(files("../libs/plugin-api.jar")) + classpath(files("../libs/gradle-plugin.jar")) + classpath("com.android.tools.build:gradle:8.11.0") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:2.3.0") + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { google(); mavenCentral() } +} + +rootProject.name = "vector-search-plugin" diff --git a/vector-search-plugin/src/main/AndroidManifest.xml b/vector-search-plugin/src/main/AndroidManifest.xml new file mode 100644 index 00000000..aecb226b --- /dev/null +++ b/vector-search-plugin/src/main/AndroidManifest.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/vector-search-plugin/src/main/assets/docs/index.html b/vector-search-plugin/src/main/assets/docs/index.html new file mode 100644 index 00000000..3d64461f --- /dev/null +++ b/vector-search-plugin/src/main/assets/docs/index.html @@ -0,0 +1,75 @@ + + + + + +Vector Search — Guide + + + +

Vector Search — Semantic Code Search

+ +

Vector Search adds meaning-based search to the project. Instead of + matching exact text, it ranks code by semantic similarity to your + query and contributes a "Semantic Results" section to the project + search screen.

+ +

How it works

+
    +
  • Project files are split into chunks and turned into embedding vectors.
    • +
  • Your query is embedded the same way and compared by cosine similarity; + the closest chunks are returned.
    • +
  • Embeddings come from AI Core when it's installed. Without it, the + plugin falls back to a lightweight lexical embedding so search still works + (with lower quality).
    • +
+ +

Using it

+

Open the project search screen and run a query as usual. When Vector Search + is active, results include a Semantic Results section alongside the + normal text matches. Indexing happens on demand for the searched roots.

+ +

Privacy

+
+ Indexing reads files in the current project only. Embeddings are stored in a + local database on the device. If AI Core's Gemini backend is selected + for embeddings, chunk text is sent to Google over HTTPS; the Local + backend and the lexical fallback keep everything on-device. +
+ +

Troubleshooting

+
    +
  • Results look weak — install/activate AI Core so real model + embeddings are used instead of the lexical fallback.
    • +
  • No semantic results — the roots may not be indexed yet; run the + search again after indexing completes.
    • +
+ + diff --git a/vector-search-plugin/src/main/assets/icon_day.png b/vector-search-plugin/src/main/assets/icon_day.png new file mode 100644 index 00000000..8d1ad7f7 Binary files /dev/null and b/vector-search-plugin/src/main/assets/icon_day.png differ diff --git a/vector-search-plugin/src/main/assets/icon_night.png b/vector-search-plugin/src/main/assets/icon_night.png new file mode 100644 index 00000000..99ffa838 Binary files /dev/null and b/vector-search-plugin/src/main/assets/icon_night.png differ diff --git a/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/CodeChunker.kt b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/CodeChunker.kt new file mode 100644 index 00000000..189de650 --- /dev/null +++ b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/CodeChunker.kt @@ -0,0 +1,563 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.plugins.vectorsearch + +import java.io.File + +/** + * Data class representing a chunked piece of code with metadata. + * + * @param content The actual text content of the chunk + * @param startLine The starting line number (0-indexed) in the original file + * @param endLine The ending line number (0-indexed, inclusive) in the original file + * @param isCodeChunk Whether this chunk contains code boundaries (functions, classes, etc.) + */ +data class CodeChunk( + val content: String, + val startLine: Int, + val endLine: Int, + val isCodeChunk: Boolean, +) + +/** + * Semantic file chunker that splits files into manageable pieces based on language-specific + * boundaries (functions, classes, etc.) with configurable overlap for context preservation. + * + * Supports: + * - Language detection from file extensions + * - Smart boundary detection for Kotlin/Java code + * - Configurable overlap between chunks to maintain context + * - Merging of very small chunks (< minChunkSize) with next chunk + * - Simple text files bypass boundary detection + */ +object CodeChunker { + // Default configuration. + // Chunks are kept small so a match localizes to a single method/block rather than a + // whole file, while staying within the embedding model's token budget. + private const val DEFAULT_MAX_CHUNK_SIZE = 600 + private const val DEFAULT_OVERLAP_SIZE = 80 + private const val DEFAULT_MIN_CHUNK_SIZE = 5 + + // Code file extensions that should use boundary detection + private val CODE_EXTENSIONS = setOf( + "kt", "java", "scala", "groovy", + "py", "js", "ts", "tsx", "jsx", + "go", "rs", "c", "cpp", "h", "hpp", + "cs", "swift", "rb", "php" + ) + + // Text file extensions that should NOT use boundary detection + private val TEXT_EXTENSIONS = setOf( + "txt", "md", "rst", "csv", "json", "xml", "yaml", "yml", + "html", "css", "sql" + ) + + /** + * Chunks a file into semantic pieces. + * + * @param file The file to chunk + * @param maxChunkSize Maximum size of each chunk in characters (default: 2000) + * @param overlapSize Number of characters to overlap between chunks (default: 100) + * @param minChunkSize Minimum chunk size before merging with next chunk (default: 5 lines) + * @return List of CodeChunk objects representing the file + * @throws IllegalArgumentException if file doesn't exist or cannot be read + */ + fun chunkFile( + file: File, + maxChunkSize: Int = DEFAULT_MAX_CHUNK_SIZE, + overlapSize: Int = DEFAULT_OVERLAP_SIZE, + minChunkSize: Int = DEFAULT_MIN_CHUNK_SIZE, + ): List { + require(file.exists()) { "File does not exist: ${file.absolutePath}" } + require(file.isFile) { "Path is not a file: ${file.absolutePath}" } + require(maxChunkSize > 0) { "maxChunkSize must be positive" } + require(overlapSize >= 0) { "overlapSize must be non-negative" } + require(minChunkSize > 0) { "minChunkSize must be positive" } + + val content = file.readText() + return chunkText( + content, + file.extension, + maxChunkSize, + overlapSize, + minChunkSize, + ) + } + + /** + * Chunks text content into semantic pieces. + * + * @param content The text content to chunk + * @param fileExtension The file extension to determine language (e.g., "kt", "java", "py") + * @param maxChunkSize Maximum size of each chunk in characters (default: 2000) + * @param overlapSize Number of characters to overlap between chunks (default: 100) + * @param minChunkSize Minimum chunk size before merging with next chunk (default: 5 lines) + * @return List of CodeChunk objects + */ + fun chunkText( + content: String, + fileExtension: String = "txt", + maxChunkSize: Int = DEFAULT_MAX_CHUNK_SIZE, + overlapSize: Int = DEFAULT_OVERLAP_SIZE, + minChunkSize: Int = DEFAULT_MIN_CHUNK_SIZE, + ): List { + if (content.isBlank()) { + return emptyList() + } + + // Strip leading license/comment headers (and XML prolog) so the shared boilerplate + // doesn't pollute embeddings or snippets. The removed line count is added back to + // each chunk's range so navigation still lands on the right line. + val (cleaned, lineOffset) = stripLeadingBoilerplate(content, fileExtension) + if (cleaned.isBlank()) { + return emptyList() + } + + val lines = cleaned.split("\n") + val isCodeFile = isCodeLanguage(fileExtension) + + // For text files or very small files, use simple line-based chunking + val chunks = if (!isCodeFile || lines.size <= 10) { + simpleChunk(lines, maxChunkSize, overlapSize) + } else { + // For code files, use boundary-aware chunking + semanticChunk(lines, maxChunkSize, overlapSize, minChunkSize) + } + + return if (lineOffset == 0) { + chunks + } else { + chunks.map { + it.copy( + startLine = it.startLine + lineOffset, + endLine = it.endLine + lineOffset, + ) + } + } + } + + /** + * Strips a leading license/comment header (for code) or XML prolog/comment (for XML) from + * [content]. Returns the cleaned content and the number of leading lines removed, so the + * caller can offset chunk line numbers back onto the original file. + * + * Only contiguous *leading* boilerplate is removed, so the offset is uniform for all + * remaining lines. If the file turns out to be nothing but boilerplate, the original + * content is kept (offset 0) to avoid dropping it entirely. + */ + private fun stripLeadingBoilerplate(content: String, fileExtension: String): Pair { + val lines = content.split("\n") + var idx = 0 + + fun skipBlankLines() { + while (idx < lines.size && lines[idx].isBlank()) idx++ + } + + skipBlankLines() + val firstNonBlank = idx + + when { + isCodeLanguage(fileExtension) -> { + // Leading block comment (e.g. license header): /* ... */ + if (idx < lines.size && lines[idx].trimStart().startsWith("/*")) { + while (idx < lines.size && !lines[idx].contains("*/")) idx++ + if (idx < lines.size) idx++ // consume the line with the closing */ + } + skipBlankLines() + // Leading single-line comments: // + while (idx < lines.size && lines[idx].trimStart().startsWith("//")) idx++ + skipBlankLines() + } + + fileExtension.lowercase() == "xml" -> { + // XML declaration: + if (idx < lines.size && lines[idx].trimStart().startsWith(" + if (idx < lines.size && lines[idx].trimStart().startsWith("")) idx++ + if (idx < lines.size) idx++ + } + skipBlankLines() + } + } + + // Nothing stripped beyond initial blanks, or the whole file was boilerplate: + // keep the original content so we never lose everything. + if (idx <= firstNonBlank || idx >= lines.size) { + return content to 0 + } + + val cleaned = lines.subList(idx, lines.size).joinToString("\n") + return cleaned to idx + } + + /** + * Simple line-based chunking without boundary detection. + * Used for text files or very small files. + */ + private fun simpleChunk( + lines: List, + maxChunkSize: Int, + overlapSize: Int, + ): List { + val chunks = mutableListOf() + var currentStart = 0 + + while (currentStart < lines.size) { + val currentChunk = mutableListOf() + var currentSize = 0 + var currentEnd = currentStart + + // Add lines until we reach maxChunkSize or run out of lines + while (currentEnd < lines.size) { + val line = lines[currentEnd] + val lineSize = line.length + 1 // +1 for newline + if (currentSize + lineSize > maxChunkSize && currentChunk.isNotEmpty()) { + break + } + currentChunk.add(line) + currentSize += lineSize + currentEnd++ + } + + if (currentChunk.isNotEmpty()) { + chunks.add( + CodeChunk( + content = currentChunk.joinToString("\n"), + startLine = currentStart, + endLine = currentEnd - 1, + isCodeChunk = false, + ) + ) + + // Move to next chunk with overlap + val overlapLines = calculateOverlapLines(currentChunk, overlapSize) + currentStart = maxOf(currentEnd - overlapLines, currentStart + 1) + } else { + // If even a single line is too large, include it anyway + if (currentEnd < lines.size) { + chunks.add( + CodeChunk( + content = lines[currentEnd], + startLine = currentEnd, + endLine = currentEnd, + isCodeChunk = false, + ) + ) + currentEnd++ + currentStart = currentEnd + } else { + break + } + } + } + + return chunks + } + + /** + * Semantic chunking for code files. + * Breaks on function/class boundaries and respects minChunkSize. + */ + private fun semanticChunk( + lines: List, + maxChunkSize: Int, + overlapSize: Int, + minChunkSize: Int, + ): List { + val chunks = mutableListOf() + var currentStart = 0 + + while (currentStart < lines.size) { + val (chunkLines, nextStart, brokeAtDeclaration) = buildSemanticChunk( + lines, + currentStart, + maxChunkSize, + ) + + if (chunkLines.isEmpty()) { + break + } + + // Expand a too-small chunk so we don't emit slivers — but never across a declaration + // boundary (that would re-absorb the next method into this chunk and undo the + // per-method localization). Only meaningful for mid-flow (size-limit) breaks. + if (chunkLines.size < minChunkSize && nextStart < lines.size && !brokeAtDeclaration) { + val expandedLines = chunkLines.toMutableList() + var expandedStart = nextStart + + while (expandedLines.size < minChunkSize && + expandedStart < lines.size && + !isDeclarationStart(lines[expandedStart]) + ) { + expandedLines.add(lines[expandedStart]) + expandedStart++ + } + + chunks.add( + CodeChunk( + content = expandedLines.joinToString("\n"), + startLine = currentStart, + endLine = currentStart + expandedLines.size - 1, + isCodeChunk = true, + ) + ) + + currentStart = expandedStart + } else { + chunks.add( + CodeChunk( + content = chunkLines.joinToString("\n"), + startLine = currentStart, + endLine = currentStart + chunkLines.size - 1, + isCodeChunk = true, + ) + ) + + // The chunk reached the end of the file: stop, otherwise the overlap step below + // would keep re-emitting shrinking fragments of the already-covered tail. + if (nextStart >= lines.size) { + break + } + + // Overlap only makes sense for a mid-flow (size-limit) break, to carry context + // into the next chunk. A clean declaration boundary starts the next chunk exactly + // at the declaration; adding overlap there just duplicates this chunk's tail. + currentStart = if (brokeAtDeclaration) { + nextStart + } else { + val overlapLines = calculateOverlapLines(chunkLines, overlapSize) + maxOf(nextStart - overlapLines, currentStart + 1) + } + } + } + + // Post-process to handle overlap: ensure last N lines of chunk N match first N lines of N+1 + return reconcileOverlaps(chunks) + } + + /** + * Builds a single semantic chunk starting from a given line. + * + * Breaks proactively BEFORE the next top-level declaration (a new method/function/class, or + * its leading annotation) so that a semantic match localizes to a single method instead of an + * entire class body. Without this, a small class whose body fits inside [maxChunkSize] never + * splits and the whole body becomes one coarse chunk. The break only fires once the chunk holds + * substantive content that is not merely the declaration's own leading annotation(s) (see + * [hasBreakableContentBefore]), so annotations stay attached to what they annotate. Falls back + * to breaking AFTER a closing brace when the size limit is hit before any declaration boundary. + * + * @return Triple of (chunk lines, next start line index, whether the break was at a declaration + * boundary — a clean boundary the caller should not apply overlap across) + */ + private fun buildSemanticChunk( + lines: List, + startLine: Int, + maxChunkSize: Int, + ): Triple, Int, Boolean> { + val chunk = mutableListOf() + var currentSize = 0 + var currentLine = startLine + var lastBoundaryLine = startLine // Last line with a closing boundary + + while (currentLine < lines.size) { + val line = lines[currentLine] + val lineSize = line.length + 1 // +1 for newline + + // Start a fresh chunk at the next declaration so each method/function is its own chunk. + if (currentLine > startLine && + isDeclarationStart(line) && + hasBreakableContentBefore(lines, startLine, currentLine) + ) { + return Triple(lines.subList(startLine, currentLine), currentLine, true) + } + + if (currentSize + lineSize > maxChunkSize && chunk.isNotEmpty()) { + // We've exceeded the size limit + // Break at the last boundary if we have one + if (lastBoundaryLine >= startLine && lastBoundaryLine < currentLine) { + return Triple( + lines.subList(startLine, lastBoundaryLine + 1), + lastBoundaryLine + 1, + false, + ) + } else { + // No boundary found, just break here + return Triple( + lines.subList(startLine, currentLine), + currentLine, + false, + ) + } + } + + chunk.add(line) + currentSize += lineSize + + // Track closing braces as potential break points (prefer breaking AFTER }) + if (line.trim().startsWith("}")) { + lastBoundaryLine = currentLine + } + + // Also track function/class/object starts as potential context boundaries + if (isBoundaryLine(line)) { + lastBoundaryLine = currentLine + } + + currentLine++ + } + + return Triple( + lines.subList(startLine, minOf(currentLine, lines.size)), + minOf(currentLine, lines.size), + false, + ) + } + + /** + * True if lines `[startLine, currentLine)` contain substantive content beyond the trailing run + * of blank and annotation lines that belong to the declaration at [currentLine]. Used so that + * we break BEFORE a declaration's leading annotations (keeping them attached) but do not break + * when the chunk so far is only those annotations. + */ + private fun hasBreakableContentBefore( + lines: List, + startLine: Int, + currentLine: Int, + ): Boolean { + var i = currentLine - 1 + while (i >= startLine && lines[i].isBlank()) i-- + while (i >= startLine && lines[i].trim().startsWith("@")) i-- + while (i >= startLine && lines[i].isBlank()) i-- + return i >= startLine + } + + /** + * Checks if a line contains a code boundary marker (fun, class, object, interface). + * Only matches meaningful declarations, not comments. + */ + private fun isBoundaryLine(line: String): Boolean { + val trimmed = line.trim() + if (trimmed.startsWith("//") || trimmed.startsWith("*") || trimmed.startsWith("/*")) { + return false + } + + // Match Kotlin/Java keywords for declarations + return trimmed.matches(KEYWORD_DECLARATION) + } + + // Kotlin/Java keyword-led declarations (class/fun/object/...). + private val KEYWORD_DECLARATION = + Regex("""^(public|private|protected|internal)?\s*(fun|class|object|interface|enum|sealed|data|companion|open|abstract)\b.*""") + + // Java-style method signature: optional modifiers, then ` (`. Two + // whitespace-separated tokens before the paren is what distinguishes a declaration + // (`void onCreate(`) from a bare call (`setContentView(`), which has only one. + private val JAVA_METHOD_SIGNATURE = + Regex( + """^(?:(?:public|private|protected|static|final|abstract|synchronized|native|default|strictfp)\s+)*""" + + """[A-Za-z_][\w.$]*(?:<[^>]*>)?(?:\[\])*\s+""" + // return type (optional generics / array) + """[A-Za-z_]\w*\s*\(.*""", // method name followed by ( + ) + + // Statements that also look like `name(...)` but are NOT declarations. + private val STATEMENT_KEYWORDS = + setOf("if", "for", "while", "switch", "catch", "synchronized", "else", "do", "try", "when", "return", "throw", "new", "assert", "yield", "case") + + /** + * True if [line] begins a new top-level declaration that should start a fresh chunk: a leading + * annotation (so it stays attached to the declaration it precedes), a Kotlin/Java keyword + * declaration (class/fun/...), or a Java-style method signature. Used to localize a semantic + * match to a single method/function rather than an entire class body. + */ + private fun isDeclarationStart(line: String): Boolean { + val trimmed = line.trim() + if (trimmed.isEmpty() || + trimmed.startsWith("//") || + trimmed.startsWith("*") || + trimmed.startsWith("/*") + ) { + return false + } + + // Annotations (@Override, @Composable, ...) precede a declaration. + if (trimmed.startsWith("@") && trimmed.getOrNull(1)?.isLetter() == true) { + return true + } + + if (trimmed.matches(KEYWORD_DECLARATION)) { + return true + } + + // Java method signature, excluding control-flow/statements that share the `name(...)` shape. + val firstToken = trimmed.takeWhile { it.isLetterOrDigit() || it == '_' } + if (firstToken in STATEMENT_KEYWORDS) { + return false + } + return trimmed.matches(JAVA_METHOD_SIGNATURE) + } + + /** + * Calculates how many lines to use for overlap based on character count. + */ + private fun calculateOverlapLines(lines: List, overlapSize: Int): Int { + var charCount = 0 + for ((index, line) in lines.withIndex()) { + charCount += line.length + 1 // +1 for newline + if (charCount >= overlapSize) { + return index + 1 + } + } + return minOf(lines.size / 4, 10) // Default: ~25% of chunk or max 10 lines + } + + /** + * Post-processes chunks to ensure overlaps are properly set up. + * The last N lines of chunk N should match the first N lines of chunk N+1. + */ + private fun reconcileOverlaps(chunks: List): List { + if (chunks.size <= 1) { + return chunks + } + + val reconciled = mutableListOf() + + for (i in chunks.indices) { + val chunk = chunks[i] + + if (i < chunks.size - 1) { + val nextChunk = chunks[i + 1] + // The overlap is implicitly handled by the ranges + // Just ensure they're properly tracked + } + + reconciled.add(chunk) + } + + return reconciled + } + + /** + * Determines if the file extension represents a code language. + */ + private fun isCodeLanguage(extension: String): Boolean { + val normalized = extension.lowercase() + return CODE_EXTENSIONS.contains(normalized) + } +} diff --git a/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/CodeEmbedding.kt b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/CodeEmbedding.kt new file mode 100644 index 00000000..2f3dcb92 --- /dev/null +++ b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/CodeEmbedding.kt @@ -0,0 +1,66 @@ +package com.itsaky.androidide.plugins.vectorsearch + +/** + * Represents a code embedding (semantic vector) stored in the database. + * + * Each CodeEmbedding corresponds to a chunk of source code from a file. + * The embedding is a 384-dimensional vector computed from the code text, + * enabling semantic search and similarity comparisons. + * + * @property key Unique identifier in format "{filePath}:{chunkIndex}" + * @property filePath Full path to the source file + * @property chunkText The actual code text for this chunk + * @property language Programming language (kotlin, java, or xml) + * @property chunkIndex Which chunk of the file this is (0-indexed) + * @property startLine Line number where chunk starts (1-indexed) + * @property endLine Line number where chunk ends (inclusive, 1-indexed) + * @property embedding The 384-dimensional embedding vector + */ +data class CodeEmbedding( + val key: String, + val filePath: String, + val chunkText: String, + val language: String, + val chunkIndex: Int, + val startLine: Int, + val endLine: Int, + val embedding: FloatArray, +) { + + /** + * Custom equality check that includes floating-point array comparison. + * Uses contentEquals for FloatArray instead of identity comparison. + */ + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as CodeEmbedding + + if (key != other.key) return false + if (filePath != other.filePath) return false + if (chunkText != other.chunkText) return false + if (language != other.language) return false + if (chunkIndex != other.chunkIndex) return false + if (startLine != other.startLine) return false + if (endLine != other.endLine) return false + if (!embedding.contentEquals(other.embedding)) return false + + return true + } + + /** + * Custom hash code that includes the embedding array. + */ + override fun hashCode(): Int { + var result = key.hashCode() + result = 31 * result + filePath.hashCode() + result = 31 * result + chunkText.hashCode() + result = 31 * result + language.hashCode() + result = 31 * result + chunkIndex + result = 31 * result + startLine + result = 31 * result + endLine + result = 31 * result + embedding.contentHashCode() + return result + } +} diff --git a/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/EmbeddingIndexingService.kt b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/EmbeddingIndexingService.kt new file mode 100644 index 00000000..31f22c8d --- /dev/null +++ b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/EmbeddingIndexingService.kt @@ -0,0 +1,187 @@ +package com.itsaky.androidide.plugins.vectorsearch + +import android.content.ContentValues +import android.database.sqlite.SQLiteDatabase +import android.database.sqlite.SQLiteOpenHelper +import android.content.Context +import android.util.Log +import java.io.File +import java.nio.ByteBuffer + +private const val TAG = "EmbeddingIndexing" + +/** + * SQLite helper for embeddings storage. + */ +class EmbeddingsDbHelper(context: Context) : SQLiteOpenHelper(context, "embeddings.db", null, 1) { + override fun onCreate(db: SQLiteDatabase) { + db.execSQL(""" + CREATE TABLE IF NOT EXISTS embeddings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + key TEXT UNIQUE, + file_path TEXT, + chunk_text TEXT, + language TEXT, + chunk_index INTEGER, + start_line INTEGER, + end_line INTEGER, + embedding BLOB + ) + """.trimIndent()) + db.execSQL("CREATE INDEX IF NOT EXISTS idx_file_path ON embeddings(file_path)") + } + + override fun onUpgrade(db: SQLiteDatabase?, oldVersion: Int, newVersion: Int) { + // No-op for now + } +} + +/** + * Service for indexing code files into embeddings stored in a local SQLite database. + * + * Walks the project tree, chunks files, generates embeddings via LlmInferenceService, + * and stores them for later semantic search. + */ +class EmbeddingIndexingService(private val context: Context) { + + private val dbHelper = EmbeddingsDbHelper(context) + + /** + * Stores a pre-computed embedding in the database. + * Call this method after generating embeddings via LlmInferenceService. + * + * @param embedding The CodeEmbedding to store + */ + fun storeEmbeddingDirect(embedding: CodeEmbedding) { + storeEmbedding(embedding) + Log.d(TAG, "Stored embedding for ${embedding.filePath}:${embedding.chunkIndex}") + } + + /** + * Collects and returns all code files in a directory that can be chunked and indexed. + * Caller is responsible for generating embeddings and storing them via storeEmbeddingDirect(). + * + * @param projectRoot Root directory of the project + * @param maxFiles Maximum number of files to collect (default 500) + * @return List of code files ready for chunking + */ + fun collectFiles(projectRoot: File, maxFiles: Int = 500): List { + return collectCodeFiles(projectRoot, maxFiles) + } + + fun languageFor(file: File): String { + return getLanguageFromExtension(file.extension) + } + + /** + * Retrieves all embeddings from the database. + */ + fun getAllEmbeddings(): List { + val db = dbHelper.readableDatabase + val embeddings = mutableListOf() + + db.query( + "embeddings", + arrayOf("key", "file_path", "chunk_text", "language", "chunk_index", "start_line", "end_line", "embedding"), + null, null, null, null, null + ).use { cursor -> + while (cursor.moveToNext()) { + val buffer = cursor.getBlob(7) + val embedding = FloatArray(buffer.size / 4) + val byteBuffer = ByteBuffer.wrap(buffer) + for (i in embedding.indices) { + embedding[i] = byteBuffer.float + } + + embeddings.add( + CodeEmbedding( + key = cursor.getString(0), + filePath = cursor.getString(1), + chunkText = cursor.getString(2), + language = cursor.getString(3), + chunkIndex = cursor.getInt(4), + startLine = cursor.getInt(5), + endLine = cursor.getInt(6), + embedding = embedding, + ) + ) + } + } + + return embeddings + } + + /** + * Clears all embeddings from the database (useful for reindexing). + */ + fun clearIndex() { + val db = dbHelper.writableDatabase + db.delete("embeddings", null, null) + Log.i(TAG, "Index cleared") + } + + private fun storeEmbedding(embedding: CodeEmbedding) { + val db = dbHelper.writableDatabase + val buffer = ByteBuffer.allocate(embedding.embedding.size * 4) + for (f in embedding.embedding) { + buffer.putFloat(f) + } + + val values = ContentValues().apply { + put("key", embedding.key) + put("file_path", embedding.filePath) + put("chunk_text", embedding.chunkText) + put("language", embedding.language) + put("chunk_index", embedding.chunkIndex) + put("start_line", embedding.startLine) + put("end_line", embedding.endLine) + put("embedding", buffer.array()) + } + + db.insertWithOnConflict("embeddings", null, values, SQLiteDatabase.CONFLICT_REPLACE) + } + + private fun collectCodeFiles(root: File, maxCount: Int): List { + val files = mutableListOf() + val skipDirs = setOf("build", ".gradle", "node_modules", ".git", "dist", "out") + + fun walk(dir: File) { + if (files.size >= maxCount) return + if (dir.name.startsWith(".") && dir.name != ".") return + if (dir.name in skipDirs) return + + try { + dir.listFiles()?.forEach { file -> + when { + file.isDirectory -> walk(file) + file.isFile && isCodeFile(file) -> files.add(file) + } + if (files.size >= maxCount) return@forEach + } + } catch (e: Exception) { + Log.w(TAG, "Error walking directory ${dir.absolutePath}", e) + } + } + + walk(root) + return files + } + + private fun isCodeFile(file: File): Boolean { + val ext = file.extension.lowercase() + return ext in setOf("kt", "java", "xml", "gradle", "kts", "py", "js", "ts") + } + + private fun getLanguageFromExtension(ext: String): String { + return when (ext.lowercase()) { + "kt", "kts" -> "kotlin" + "java" -> "java" + "xml" -> "xml" + "gradle" -> "gradle" + "py" -> "python" + "js", "jsx" -> "javascript" + "ts", "tsx" -> "typescript" + else -> "text" + } + } +} diff --git a/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorMath.kt b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorMath.kt new file mode 100644 index 00000000..e6b18efa --- /dev/null +++ b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorMath.kt @@ -0,0 +1,57 @@ +package com.itsaky.androidide.plugins.vectorsearch + +import kotlin.math.sqrt + +/** + * Utility object for vector mathematics operations. + * Provides functions for comparing and analyzing embeddings. + */ +object VectorMath { + + /** + * Calculates the cosine similarity between two vectors. + * + * Cosine similarity is the standard metric for comparing embeddings. + * It measures the cosine of the angle between two vectors in a multi-dimensional space. + * + * Algorithm: + * 1. Calculate dot product: sum of a[i] * b[i] + * 2. Calculate norm for vector a: sqrt(sum of a[i]²) + * 3. Calculate norm for vector b: sqrt(sum of b[i]²) + * 4. Return dotProduct / (normA * normB) + * 5. Handle zero-division: return 0.0f if denominator is 0 + * + * @param a First embedding vector + * @param b Second embedding vector + * @return Float between -1.0 and 1.0 (or 0.0 if either vector has zero magnitude) + * - 1.0 indicates identical direction (maximum similarity) + * - 0.0 indicates orthogonal vectors (no similarity) + * - -1.0 indicates opposite direction (inverse similarity) + */ + fun cosineSimilarity(a: FloatArray, b: FloatArray): Float { + var dotProduct = 0.0f + var normASquared = 0.0f + var normBSquared = 0.0f + + // Single-pass calculation for efficiency + for (i in a.indices) { + val aVal = a[i] + val bVal = b[i] + + dotProduct += aVal * bVal + normASquared += aVal * aVal + normBSquared += bVal * bVal + } + + val normA = sqrt(normASquared) + val normB = sqrt(normBSquared) + val denominator = normA * normB + + // Handle zero-division case + return if (denominator == 0.0f) { + 0.0f + } else { + dotProduct / denominator + } + } +} diff --git a/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorSearchPlugin.kt b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorSearchPlugin.kt new file mode 100644 index 00000000..da8b33b4 --- /dev/null +++ b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorSearchPlugin.kt @@ -0,0 +1,302 @@ +package com.itsaky.androidide.plugins.vectorsearch + +import android.util.Log +import com.itsaky.androidide.plugins.IPlugin +import com.itsaky.androidide.plugins.PluginContext +import com.itsaky.androidide.plugins.extensions.DocumentationExtension +import com.itsaky.androidide.plugins.extensions.PluginTooltipButton +import com.itsaky.androidide.plugins.extensions.PluginTooltipEntry +import com.itsaky.androidide.plugins.extensions.ProjectSearchExtension +import com.itsaky.androidide.plugins.extensions.ProjectSearchRequest +import com.itsaky.androidide.plugins.extensions.ProjectSearchResult +import com.itsaky.androidide.plugins.extensions.ProjectSearchSection +import com.itsaky.androidide.plugins.services.LlmInferenceService +import com.itsaky.androidide.plugins.services.SharedServices +import kotlinx.coroutines.runBlocking +import java.io.File +import java.util.concurrent.CompletableFuture +import kotlin.math.sqrt + +private const val TAG = "VectorSearchPlugin" +private const val AI_CORE_PLUGIN_ID = "com.itsaky.androidide.plugins.aicore" +private const val SEMANTIC_RESULTS_TITLE = "Semantic Results" +private const val FALLBACK_EMBEDDING_DIMENSIONS = 384 + +/** + * Vector Search Plugin provides semantic code search capabilities. + * + * When activated, it indexes the project using on-device LLM embeddings + * and stores them in a local SQLite database. Other plugins can use + * this service to perform semantic searches. + */ +class VectorSearchPlugin : IPlugin, ProjectSearchExtension, DocumentationExtension { + + private lateinit var context: PluginContext + private lateinit var indexingService: EmbeddingIndexingService + @Volatile private var indexedRootsKey: String? = null + + override fun initialize(context: PluginContext): Boolean { + this.context = context + Log.i(TAG, "VectorSearchPlugin initialized") + return true + } + + override fun activate(): Boolean { + Log.i(TAG, "VectorSearchPlugin activating...") + + // Initialize indexing service + indexingService = EmbeddingIndexingService(context.androidContext) + + Log.i(TAG, "VectorSearchPlugin activated. Call indexProject() to start indexing.") + return true + } + + override fun deactivate(): Boolean { + Log.i(TAG, "VectorSearchPlugin deactivating") + return true + } + + override fun dispose() { + Log.i(TAG, "VectorSearchPlugin disposed") + } + + /** + * Initiates project indexing (files will need to be chunked and embeddings generated separately). + * Should be called after activate(). + */ + fun prepareIndexing() { + Log.i(TAG, "Project indexing prepared. Use getFiles() and storeEmbedding() to index.") + } + + /** + * Gets the list of code files to index in the current project. + */ + fun getFiles(): List { + val projectDir = java.io.File( + System.getProperty("project.dir") ?: System.getProperty("user.dir") ?: return emptyList() + ) + return indexingService.collectFiles(projectDir) + } + + override fun searchProject(request: ProjectSearchRequest): CompletableFuture> { + Log.i(TAG, "Project search requested for '${request.query}'") + return CompletableFuture.supplyAsync { + val results = runBlocking { + search( + query = request.query, + backendId = "local", + roots = request.roots, + topK = 10, + ) + } + if (results.isEmpty()) { + Log.i(TAG, "No semantic results available for '${request.query}'") + emptyList() + } else { + listOf( + ProjectSearchSection( + title = SEMANTIC_RESULTS_TITLE, + results = results.map { it.toProjectSearchResult(request.query) }, + ) + ) + } + } + } + + /** + * Searches embeddings semantically. + * Requires the project to have been indexed first. + * + * @param query Search query string + * @param backendId LLM backend to use (default "local") + * @param topK Maximum number of results to return (default 10) + * @return List of CodeEmbedding results ranked by relevance + */ + suspend fun search( + query: String, + backendId: String = "local", + roots: List = emptyList(), + topK: Int = 10, + ): List { + return try { + val llmService = SharedServices.get(LlmInferenceService::class.java) + ?: context.getPluginService(AI_CORE_PLUGIN_ID, LlmInferenceService::class.java) + ?: context.services.get(LlmInferenceService::class.java) + + if (roots.isNotEmpty()) { + ensureIndexed(roots, llmService, backendId) + } + + val queryEmbedding = generateEmbedding(query, llmService, backendId) + val allEmbeddings = indexingService.getAllEmbeddings() + if (allEmbeddings.isEmpty()) { + Log.w(TAG, "No embeddings indexed yet") + return emptyList() + } + + val results = VectorSearchService.searchWithScores(queryEmbedding, allEmbeddings, topK = topK) + Log.d(TAG, "Search for '$query' returned ${results.size} results") + + results.map { it.first } + } catch (e: Exception) { + Log.e(TAG, "Error during search", e) + emptyList() + } + } + + /** + * Clears the index (e.g., for reindexing after project changes). + */ + fun clearIndex() { + indexingService.clearIndex() + indexedRootsKey = null + Log.i(TAG, "Index cleared") + } + + private fun ensureIndexed( + roots: List, + llmService: LlmInferenceService?, + backendId: String, + ) { + val rootsKey = roots + .map { it.absolutePath } + .sorted() + .joinToString("|") + if (indexedRootsKey == rootsKey && indexingService.getAllEmbeddings().isNotEmpty()) { + return + } + + indexingService.clearIndex() + var fileCount = 0 + var chunkCount = 0 + + roots.forEach { root -> + indexingService.collectFiles(root).forEach { file -> + fileCount++ + val language = indexingService.languageFor(file) + val chunks = try { + CodeChunker.chunkFile(file) + } catch (e: Exception) { + Log.w(TAG, "Failed to chunk ${file.absolutePath}", e) + emptyList() + } + + chunks.forEachIndexed { index, chunk -> + val embedding = CodeEmbedding( + key = "${file.absolutePath}:$index", + filePath = file.absolutePath, + chunkText = chunk.content, + language = language, + chunkIndex = index, + startLine = chunk.startLine + 1, + endLine = chunk.endLine + 1, + embedding = generateEmbedding(chunk.content, llmService, backendId), + ) + indexingService.storeEmbeddingDirect(embedding) + chunkCount++ + } + } + } + + indexedRootsKey = rootsKey + Log.i(TAG, "Indexed $chunkCount chunks from $fileCount files") + } + + private fun generateEmbedding( + text: String, + llmService: LlmInferenceService?, + backendId: String, + ): FloatArray { + val llmEmbedding = try { + llmService?.getEmbeddings(text, backendId)?.get() + } catch (e: Exception) { + Log.w(TAG, "LLM embeddings unavailable, using lexical fallback", e) + null + } + return if (llmEmbedding != null && llmEmbedding.isNotEmpty()) { + llmEmbedding + } else { + lexicalEmbedding(text) + } + } + + private fun lexicalEmbedding(text: String): FloatArray { + val vector = FloatArray(FALLBACK_EMBEDDING_DIMENSIONS) + lexicalTokens(text).forEach { token -> + val slot = (token.hashCode() and Int.MAX_VALUE) % vector.size + vector[slot] += 1f + } + + var normSquared = 0f + for (value in vector) { + normSquared += value * value + } + if (normSquared == 0f) { + return vector + } + + val norm = sqrt(normSquared) + for (i in vector.indices) { + vector[i] = vector[i] / norm + } + return vector + } + + private fun lexicalTokens(text: String): Sequence { + val words = Regex("[A-Za-z_][A-Za-z0-9_]*|\\d+") + .findAll(text) + .map { it.value.lowercase() } + return words.flatMap { word -> + sequence { + yield(word) + val maxPrefix = word.length.coerceAtMost(8) + for (length in 3..maxPrefix) { + yield(word.substring(0, length)) + } + } + } + } + + private fun CodeEmbedding.toProjectSearchResult(query: String): ProjectSearchResult { + val line = startLine.coerceAtLeast(1) - 1 + return ProjectSearchResult( + file = java.io.File(filePath), + linePreview = chunkText.replace(Regex("\\s+"), " ").take(160), + matchText = query, + startLine = line, + startColumn = 0, + endLine = endLine.coerceAtLeast(startLine).coerceAtLeast(1) - 1, + endColumn = 0, + ) + } + + override fun getTooltipCategory(): String = "plugin_vector_search" + + override fun getTooltipEntries(): List = listOf( + PluginTooltipEntry( + tag = TOOLTIP_TAG_PLUGIN, + summary = "Vector Search adds semantic, meaning-based matches to project search.", + detail = """ +

Vector Search chunks project files, generates + embeddings, and ranks matches by semantic similarity instead of + only exact text.

+

AI Core provides model embeddings when available. If it is not + loaded, Vector Search falls back to local lexical embeddings with + lower result quality.

+ """.trimIndent(), + buttons = listOf( + PluginTooltipButton( + description = "Vector Search guide", + uri = "index.html", + order = 0 + ) + ) + ) + ) + + override fun getTier3DocsAssetPath(): String = "docs" + + private companion object { + const val TOOLTIP_TAG_PLUGIN = "plugin_vector_search" + } +} diff --git a/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorSearchService.kt b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorSearchService.kt new file mode 100644 index 00000000..a9789b37 --- /dev/null +++ b/vector-search-plugin/src/main/kotlin/com/itsaky/androidide/plugins/vectorsearch/VectorSearchService.kt @@ -0,0 +1,49 @@ +package com.itsaky.androidide.plugins.vectorsearch + +/** + * Service for searching code embeddings by semantic similarity. + * Given a query string's embedding, finds the most relevant stored embeddings. + */ +object VectorSearchService { + + /** + * Searches embeddings by cosine similarity to a query embedding. + * + * @param queryEmbedding The embedding vector for the search query + * @param allEmbeddings List of all stored embeddings to search + * @param similarityThreshold Minimum similarity score (0.0-1.0) to include in results (default 0.05) + * @param topK Maximum number of results to return (default 10) + * @return List of (embedding, score) pairs sorted by similarity (highest first) + */ + fun searchWithScores( + queryEmbedding: FloatArray, + allEmbeddings: List, + similarityThreshold: Float = 0.05f, + topK: Int = 10, + ): List> { + if (allEmbeddings.isEmpty() || queryEmbedding.isEmpty()) { + return emptyList() + } + + // Compute cosine similarity for each embedding + val similarities = allEmbeddings.map { embedding -> + Pair(embedding, VectorMath.cosineSimilarity(queryEmbedding, embedding.embedding)) + } + + // Filter by threshold + val filtered = similarities.filter { it.second >= similarityThreshold } + + // Apply relevance cutoff: keep results within 50% of top score and above 0.1 floor + if (filtered.isEmpty()) { + return emptyList() + } + + val topScore = filtered.maxByOrNull { it.second }?.second ?: return emptyList() + val relevanceThreshold = (topScore * 0.5f).coerceAtLeast(0.1f) + + return filtered + .filter { it.second >= relevanceThreshold } + .sortedByDescending { it.second } + .take(topK) + } +} diff --git a/vector-search-plugin/src/main/res/values/colors.xml b/vector-search-plugin/src/main/res/values/colors.xml new file mode 100644 index 00000000..03363c00 --- /dev/null +++ b/vector-search-plugin/src/main/res/values/colors.xml @@ -0,0 +1,8 @@ + + + + #485D92 + #FFFFFF + #DAE2FF + #001847 + diff --git a/vector-search-plugin/src/main/res/values/styles.xml b/vector-search-plugin/src/main/res/values/styles.xml new file mode 100644 index 00000000..17bb8c55 --- /dev/null +++ b/vector-search-plugin/src/main/res/values/styles.xml @@ -0,0 +1,9 @@ + + + +