package net.mezimmah.wkt9 import android.annotation.SuppressLint import android.content.Intent import android.inputmethodservice.InputMethodService import android.media.MediaRecorder import android.text.InputType import android.util.Log import android.view.KeyEvent import android.view.View import android.view.ViewConfiguration import android.view.inputmethod.EditorInfo import android.view.inputmethod.InputMethodManager import android.view.textservice.SentenceSuggestionsInfo import android.view.textservice.SpellCheckerSession import android.view.textservice.SuggestionsInfo import android.widget.LinearLayout import android.widget.TextView import android.widget.Toast import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay import kotlinx.coroutines.launch import net.mezimmah.wkt9.dao.SettingDao import net.mezimmah.wkt9.dao.WordDao import net.mezimmah.wkt9.db.AppDatabase import net.mezimmah.wkt9.inputmode.InputMode import net.mezimmah.wkt9.inputmode.AlphaInputMode import net.mezimmah.wkt9.inputmode.FNInputMode import net.mezimmah.wkt9.inputmode.IdleInputMode import net.mezimmah.wkt9.inputmode.NumericInputMode import net.mezimmah.wkt9.inputmode.Status import net.mezimmah.wkt9.inputmode.WordInputMode import net.mezimmah.wkt9.inputmode.WKT9InputMode import net.mezimmah.wkt9.keypad.KeyCodeMapping import net.mezimmah.wkt9.keypad.KeyEventResult import net.mezimmah.wkt9.keypad.KeyLayout import net.mezimmah.wkt9.keypad.Keypad import net.mezimmah.wkt9.t9.T9 import net.mezimmah.wkt9.voice.Whisper import okio.IOException import java.io.File import java.lang.StringBuilder import java.util.Locale //val info = arrayOf(TextInfo("banan#", 0, 6, 0, 0)) // //spellCheckerSession?.getSentenceSuggestions(info, 10) class WKT9: InputMethodService(), SpellCheckerSession.SpellCheckerSessionListener { private val tag = "WKT9" // Dao - Database private lateinit var db: AppDatabase private lateinit var wordDao: WordDao private lateinit var settingDao: SettingDao // Coroutines private val queryScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) private var queryJob: Job? = null private val ioScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private var ioJob: Job? = null private val commitScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) private var commitJob: Job? = null private var cursorPosition = 0 private var longPressTimeout = 700 // Keypad private lateinit var keypad: Keypad // T9 private lateinit var t9: T9 // Input private var languageTag = "en_US" private var lastInputMode: WKT9InputMode = WKT9InputMode.WORD private var inputMode: InputMode? = null private lateinit var alphaInputMode: AlphaInputMode private lateinit var fnInputMode: FNInputMode private lateinit var numericInputMode: NumericInputMode private lateinit var wordInputMode: WordInputMode private lateinit var idleInputMode: IdleInputMode private var composing = false private val candidates: MutableList = mutableListOf() private var candidateIndex = 0 private var inputStatus: Status = Status.CAP private var timeout: Int? = null private var lastComposedString: String? = null private val commitHistory: MutableList = mutableListOf() // Spell checker private lateinit var locale: Locale private var allowSuggestions = false private var spellCheckerSession: SpellCheckerSession? = null // UI private lateinit var inputView: View private var toast: Toast? = null // Whisper private val whisper: Whisper = Whisper() private var recorder: MediaRecorder? = null private var recording: File? = null override fun onCreate() { val inputMethodManager = getSystemService(INPUT_METHOD_SERVICE) as InputMethodManager val inputMethodSubtype = inputMethodManager.currentInputMethodSubtype locale = inputMethodSubtype?.let { Locale.forLanguageTag(it.languageTag) } ?: Locale.forLanguageTag("en-US") Log.d(tag, "WKT9 is loading: $locale") db = AppDatabase.getInstance(this) wordDao = db.getWordDao() settingDao = db.getSettingDao() keypad = Keypad(KeyCodeMapping(KeyCodeMapping.basic), KeyLayout.en_US, KeyLayout.numeric) t9 = T9(this, keypad, settingDao, wordDao) alphaInputMode = AlphaInputMode() fnInputMode = FNInputMode() numericInputMode = NumericInputMode() wordInputMode = WordInputMode() idleInputMode = IdleInputMode() longPressTimeout = ViewConfiguration.getLongPressTimeout() lastComposedString = null commitHistory.clear() t9.initializeWords(languageTag) super.onCreate() } @SuppressLint("InflateParams") override fun onCreateInputView(): View { inputView = layoutInflater.inflate(R.layout.suggestions, null) return inputView } override fun onFinishInput() { super.onFinishInput() clearCandidates() spellCheckerSession?.cancel() spellCheckerSession?.close() inputMode = null cursorPosition = 0 inputStatus = Status.CAP spellCheckerSession = null } override fun onFinishInputView(finishingInput: Boolean) { super.onFinishInputView(finishingInput) clearCandidates() } override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { val key = keypad.getKey(keyCode) ?: return super.onKeyDown(keyCode, event) val repeatCount = event?.repeatCount ?: 0 return inputMode?.let { val keyEventResult = if (repeatCount > 0) it.onKeyDownRepeatedly(key, repeatCount, composing) else { event?.startTracking() it.onKeyDown(key, composing) } handleKeyEventResult(keyEventResult) } ?: super.onKeyDown(keyCode, event) } override fun onKeyUp(keyCode: Int, event: KeyEvent?): Boolean { val key = keypad.getKey(keyCode) ?: return super.onKeyUp(keyCode, event) val eventTime = event?.eventTime ?: 0 val downTime = event?.downTime ?: 0 val keyDownMS = eventTime - downTime return inputMode?.let { val keyEventResult = if (keyDownMS >= longPressTimeout) it.afterKeyLongDown(key, keyDownMS, composing) else it.afterKeyDown(key, composing) handleKeyEventResult(keyEventResult) } ?: super.onKeyUp(keyCode, event) } override fun onKeyLongPress(keyCode: Int, event: KeyEvent?): Boolean { val key = keypad.getKey(keyCode) ?: return super.onKeyLongPress(keyCode, event) return inputMode?.let { val keyEventResult = it.onKeyLongDown(key, composing) handleKeyEventResult(keyEventResult) } ?: super.onKeyLongPress(keyCode, event) } override fun onStartInput(attribute: EditorInfo?, restarting: Boolean) { val inputType = attribute?.inputType val inputClass = inputType?.and(InputType.TYPE_MASK_CLASS) ?: 0 val typeVariation = inputType?.and(InputType.TYPE_MASK_VARIATION) ?: 0 val typeFlags = inputType?.and(InputType.TYPE_MASK_FLAGS) ?: 0 // val textServiceManager = getSystemService(TEXT_SERVICES_MANAGER_SERVICE) as TextServicesManager cursorPosition = attribute?.initialSelEnd ?: 0 // spellCheckerSession = textServiceManager.newSpellCheckerSession(null, locale, this, false) when (inputClass) { InputType.TYPE_CLASS_DATETIME, InputType.TYPE_CLASS_NUMBER, InputType.TYPE_CLASS_PHONE -> enableInputMode(WKT9InputMode.NUMERIC) InputType.TYPE_CLASS_TEXT -> enableTextInputMode(typeVariation, typeFlags) else -> enableInputMode(WKT9InputMode.IDLE) } updateInputStatus() super.onStartInput(attribute, restarting) } override fun onUpdateSelection( oldSelStart: Int, oldSelEnd: Int, newSelStart: Int, newSelEnd: Int, candidatesStart: Int, candidatesEnd: Int ) { cursorPosition = newSelEnd super.onUpdateSelection( oldSelStart, oldSelEnd, newSelStart, newSelEnd, candidatesStart, candidatesEnd ) } override fun onGetSuggestions(p0: Array?) { TODO("Not yet implemented") } override fun onGetSentenceSuggestions(suggestionsInfo: Array?) { suggestionsInfo?.map { val suggestions = it.getSuggestionsInfoAt(0) for (index in 0 until suggestions.suggestionsCount) { val suggestion = suggestions.getSuggestionAt(index) Log.d(tag, "Suggestion: $suggestion") } } } private fun candidatesToLowerCase() { candidates.forEachIndexed { index, candidate -> candidates[index] = candidate.lowercase() } } private fun candidatesToUpperCase() { candidates.forEachIndexed { index, candidate -> candidates[index] = candidate.uppercase() } } private fun capitalizeCandidates() { candidates.forEachIndexed { index, candidate -> candidates[index] = candidate.lowercase().replaceFirstChar { it.uppercase() } } } private fun clearCandidates() { clearCandidateUI() candidates.clear() candidateIndex = 0 } private fun clearCandidateUI() { val candidatesView = inputView.findViewById(R.id.suggestions) candidatesView.removeAllViews() } private fun commitText(text: CharSequence, start: Int, end: Int): Boolean { return (markComposingRegion(start, end) && composeText(text, 1) && finishComposingText()) } private fun composeText(text: CharSequence, cursorPosition: Int = 1): Boolean { if (!composing) return false lastComposedString = text.toString() return currentInputConnection?.setComposingText(text, cursorPosition) ?: false } private fun deleteText(beforeCursor: Int, afterCursor: Int) { currentInputConnection?.deleteSurroundingText(beforeCursor, afterCursor) updateInputStatus() } // Todo: inputType private fun enableInputMode(mode: WKT9InputMode) { if (mode != WKT9InputMode.FN) lastInputMode = mode inputMode = when(mode) { WKT9InputMode.ALPHA -> alphaInputMode WKT9InputMode.NUMERIC -> numericInputMode WKT9InputMode.WORD -> wordInputMode WKT9InputMode.FN -> fnInputMode else -> idleInputMode } } private fun enableTextInputMode(variation: Int, flags: Int) { val letterVariations = listOf( InputType.TYPE_TEXT_VARIATION_EMAIL_ADDRESS, InputType.TYPE_TEXT_VARIATION_URI, InputType.TYPE_TEXT_VARIATION_WEB_EMAIL_ADDRESS, InputType.TYPE_TEXT_VARIATION_PASSWORD, InputType.TYPE_TEXT_VARIATION_FILTER, InputType.TYPE_TEXT_VARIATION_VISIBLE_PASSWORD, InputType.TYPE_TEXT_VARIATION_WEB_PASSWORD, InputType.TYPE_TEXT_VARIATION_POSTAL_ADDRESS, InputType.TYPE_TEXT_VARIATION_PERSON_NAME ) if (letterVariations.contains(variation)) { allowSuggestions = false enableInputMode(WKT9InputMode.ALPHA) } else if (lastInputMode == WKT9InputMode.ALPHA) { allowSuggestions = flags != InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS enableInputMode(WKT9InputMode.ALPHA) } else enableInputMode(WKT9InputMode.WORD) } private fun finishComposingText(): Boolean { return if (composing) { composing = false lastComposedString?.let { commitHistory.add(it) lastComposedString = null } if (allowSuggestions) Log.d(tag, "History: $commitHistory") updateInputStatus() currentInputConnection?.finishComposingText() ?: false } else false } @SuppressLint("DiscouragedApi") private fun getIconResource(): Int { val mode = inputMode?.mode ?: return resources.getIdentifier("wkt9", "drawable", packageName) val name = mode.plus("_") .plus(locale.toString()) .plus("_") .plus(inputStatus.toString()) .replace('-', '_') .lowercase() Log.d(tag, name) return resources.getIdentifier(name, "drawable", packageName) } private fun goHome() { with(Intent(Intent.ACTION_MAIN)) { this.addCategory(Intent.CATEGORY_HOME) this.flags = Intent.FLAG_ACTIVITY_NEW_TASK startActivity(this) } } private fun handleComposeTimeout(timeout: Int?) { this.timeout = timeout commitJob?.cancel() if (timeout == null) return commitJob = commitScope.launch { delay(timeout.toLong()) finishComposingText() clearCandidates() } } private fun handleKeyEventResult(res: KeyEventResult): Boolean { if (res.finishComposing) finishComposingText() if (res.startComposing) markComposingRegion() if (res.increaseWeight) onIncreaseWeight() if (!res.codeWord.isNullOrEmpty()) onCodeWordUpdate(res.codeWord, res.timeout) if (!res.candidates.isNullOrEmpty()) onCandidates(res.candidates, res.timeout) if (!res.commit.isNullOrEmpty()) onCommit(res.commit) if (res.deleteBeforeCursor > 0 || res.deleteAfterCursor > 0) onDelete(res.deleteBeforeCursor, res.deleteAfterCursor) if (res.goHome) goHome() if (res.left) onLeft() if (res.right) onRight() if (res.record) onRecord() if (res.transcribe) onTranscribe() if (res.updateInputStatus) updateInputStatus() if (res.updateWordStatus) onUpdateWordStatus() if (res.focus) onFocus() if (res.switchInputMode != null) onSwitchInputMode(res.switchInputMode) if (res.functionMode) onFunctionMode() return res.consumed } private fun isSentenceStart(): Boolean { if (cursorPosition == 0) return true val textBeforeCursor = currentInputConnection?.getTextBeforeCursor(10, 0) ?: return false if ( textBeforeCursor.trimEnd().isEmpty() || listOf('.', '!', '?').contains(textBeforeCursor.trimEnd().last())) return true return false } private fun loadCandidates(highLight: Int? = null) { val candidatesView = inputView.findViewById(R.id.suggestions) candidates.forEachIndexed { index, candidate -> val layout = if (index == highLight) R.layout.current_suggestion else R.layout.suggestion val candidateView = layoutInflater.inflate(layout, null) val textView = candidateView.findViewById(R.id.suggestion_text) textView.text = candidate candidatesView.addView(candidateView) } } private fun markComposingRegion(start: Int? = null, end: Int? = null): Boolean { if (composing) return false val composeStart = start ?: cursorPosition val composeEnd = end ?: cursorPosition composing = currentInputConnection?.setComposingRegion(composeStart, composeEnd) ?: false return composing } private fun onCandidates(candidates: List, timeout: Int?) { clearCandidates() candidates.forEach { val candidate = when (inputStatus) { Status.CAP -> it.replaceFirstChar { char -> char.uppercase() } Status.UPPER -> it.uppercase() else -> it } this.candidates.add(candidate) } loadCandidates(candidateIndex) composeText(this.candidates[candidateIndex]) handleComposeTimeout(timeout) } private fun onCodeWordUpdate(codeWord: StringBuilder, timeout: Int?) { clearCandidates() queryJob?.cancel() queryJob = queryScope.launch { val hasCandidates = queryT9Candidates(codeWord, 10) if (!hasCandidates) return@launch loadCandidates(candidateIndex) composeText(candidates[candidateIndex], 1) handleComposeTimeout(timeout) } } private fun onCommit(text: String) { commitText(text, cursorPosition, cursorPosition) } private fun onDelete(beforeCursor: Int, afterCursor: Int) { clearCandidates() deleteText(beforeCursor, afterCursor) } private fun onFocus() { requestShowSelf(InputMethodManager.SHOW_IMPLICIT) } private fun onFunctionMode() { enableInputMode(WKT9InputMode.FN) updateInputStatus() } private fun onIncreaseWeight() { val word = commitHistory.last() if (word.isEmpty()) return queryScope.launch { wordDao.increaseWeight(word) } } private fun onLeft() { if (candidates.isEmpty()) return candidateIndex-- if (candidateIndex < 0) candidateIndex = candidates.count() - 1 clearCandidateUI() loadCandidates(candidateIndex) composeText(candidates[candidateIndex]) handleComposeTimeout(this.timeout) } @Suppress("DEPRECATION") private fun onRecord() { // The recorder must be busy... if (recorder !== null || !isInputViewShown) return clearCandidates() // Delete possible existing recording recording?.delete() // Toast settings val text = "Recording now.\nRelease the button to start transcribing." val duration = Toast.LENGTH_SHORT // Instantiate recorder and start recording recorder = MediaRecorder().also { recording = File.createTempFile("recording.3gp", null, cacheDir) it.setAudioSource(MediaRecorder.AudioSource.MIC) it.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP) it.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB) it.setOutputFile(recording) try { it.prepare() it.start() toast?.cancel() toast = Toast.makeText(this, text, duration).apply { this.show() } } catch (e: Exception) { Log.d(tag, "Failed to start recording", e) } } } private fun onRight() { if (candidates.isEmpty()) return candidateIndex++ if (candidateIndex >= candidates.count()) candidateIndex = 0 clearCandidateUI() loadCandidates(candidateIndex) composeText(candidates[candidateIndex]) handleComposeTimeout(this.timeout) } private fun onSwitchInputMode(mode: WKT9InputMode) { when (mode) { WKT9InputMode.ALPHA -> enableInputMode(WKT9InputMode.ALPHA) WKT9InputMode.NUMERIC -> enableInputMode(WKT9InputMode.NUMERIC) else -> enableInputMode(WKT9InputMode.WORD) } clearCandidates() updateInputStatus() } private fun onTranscribe() { val recorder = this.recorder ?: return recorder.stop() recorder.reset() recorder.release() this.recorder = null val text = "Sending recording to speech-to-text server for transcription." val duration = Toast.LENGTH_SHORT toast?.cancel() toast = Toast.makeText(this, text, duration).apply { this.show() } ioJob?.cancel() ioJob = ioScope.launch { try { val transcription = whisper.run(recording!!) commitText(transcription.plus(" "), cursorPosition, cursorPosition) } catch (e: IOException) { Log.d(tag, "A failure occurred in the communication with the speech-to-text server", e) } } } private fun onUpdateWordStatus() { clearCandidateUI() when (inputStatus) { Status.CAP -> { inputStatus = Status.UPPER candidatesToUpperCase() } Status.UPPER -> { inputStatus = Status.LOWER candidatesToLowerCase() } else -> { inputStatus = Status.CAP capitalizeCandidates() } } showStatusIcon(getIconResource()) loadCandidates(candidateIndex) composeText(candidates[candidateIndex]) } private fun updateInputStatus() { inputStatus = inputMode?.status ?: Status.CAP if (inputStatus == Status.CAP && !isSentenceStart()) inputStatus = Status.LOWER showStatusIcon(getIconResource()) } private suspend fun queryT9Candidates(codeWord: StringBuilder, limit: Int = 10): Boolean { val words = wordDao.findCandidates(codeWord.toString(), limit) words.forEach { word -> val candidate = when (inputStatus) { Status.CAP -> word.word.replaceFirstChar { it.uppercase() } Status.UPPER -> word.word.uppercase() else -> word.word } candidates.add(candidate) } return words.isNotEmpty() } }