package net.mezimmah.wkt9 import android.annotation.SuppressLint import android.content.Intent import android.inputmethodservice.InputMethodService import android.media.MediaRecorder import android.text.InputType import android.util.Log import android.view.KeyEvent import android.view.View import android.view.ViewConfiguration import android.view.inputmethod.EditorInfo import android.view.inputmethod.InputMethodManager import android.widget.LinearLayout import android.widget.TextView import android.widget.Toast import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.delay import kotlinx.coroutines.launch import net.mezimmah.wkt9.dao.SettingDao import net.mezimmah.wkt9.dao.WordDao import net.mezimmah.wkt9.db.AppDatabase import net.mezimmah.wkt9.inputmode.InputMode import net.mezimmah.wkt9.inputmode.AlphaInputMode import net.mezimmah.wkt9.inputmode.NumericInputMode import net.mezimmah.wkt9.inputmode.Status import net.mezimmah.wkt9.inputmode.WordInputMode import net.mezimmah.wkt9.inputmode.WKT9InputMode import net.mezimmah.wkt9.keypad.KeyCodeMapping import net.mezimmah.wkt9.keypad.KeyEventResult import net.mezimmah.wkt9.keypad.KeyLayout import net.mezimmah.wkt9.keypad.Keypad import net.mezimmah.wkt9.t9.T9 import net.mezimmah.wkt9.voice.Whisper import okio.IOException import java.io.File import java.lang.StringBuilder class WKT9: InputMethodService() { private val tag = "WKT9" // Dao - Database private lateinit var db: AppDatabase private lateinit var wordDao: WordDao private lateinit var settingDao: SettingDao // Coroutines private val queryScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) private var queryJob: Job? = null private val ioScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private var ioJob: Job? = null private val commitScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) private var commitJob: Job? = null private var cursorPosition = 0 private var longPressTimeout = 700 // Keypad private lateinit var keypad: Keypad // T9 private lateinit var t9: T9 // Input private var languageTag = "en_US" private var lastInputMode: WKT9InputMode = WKT9InputMode.WORD private var inputMode: InputMode? = null private lateinit var alphaInputMode: AlphaInputMode private lateinit var numericInputMode: NumericInputMode private lateinit var wordInputMode: WordInputMode private var composing = false private val candidates: MutableList = mutableListOf() private var candidateIndex = 0 private var inputStatus: Status = Status.CAP private var timeout: Int? = null // UI private lateinit var inputView: View private var toast: Toast? = null // Whisper private val whisper: Whisper = Whisper() private var recorder: MediaRecorder? = null private var recording: File? = null override fun onCreate() { Log.d(tag, "WKT9 is loading") db = AppDatabase.getInstance(this) wordDao = db.getWordDao() settingDao = db.getSettingDao() keypad = Keypad(KeyCodeMapping(KeyCodeMapping.basic), KeyLayout.en_US) t9 = T9(this, keypad, settingDao, wordDao) alphaInputMode = AlphaInputMode() numericInputMode = NumericInputMode() wordInputMode = WordInputMode() longPressTimeout = ViewConfiguration.getLongPressTimeout() t9.initializeWords(languageTag) super.onCreate() } @SuppressLint("InflateParams") override fun onCreateInputView(): View { inputView = layoutInflater.inflate(R.layout.suggestions, null) return inputView } override fun onFinishInput() { super.onFinishInput() inputMode = null cursorPosition = 0 inputStatus = Status.CAP } override fun onFinishInputView(finishingInput: Boolean) { super.onFinishInputView(finishingInput) clearCandidates() } override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean { val key = keypad.getKey(keyCode) ?: return super.onKeyDown(keyCode, event) val repeatCount = event?.repeatCount ?: 0 return inputMode?.let { val keyEventResult = if (repeatCount > 0) it.onKeyDownRepeatedly(key, repeatCount, composing) else { event?.startTracking() it.onKeyDown(key, composing) } handleKeyEventResult(keyEventResult) } ?: super.onKeyDown(keyCode, event) } override fun onKeyUp(keyCode: Int, event: KeyEvent?): Boolean { val key = keypad.getKey(keyCode) ?: return super.onKeyUp(keyCode, event) val eventTime = event?.eventTime ?: 0 val downTime = event?.downTime ?: 0 val keyDownMS = eventTime - downTime return inputMode?.let { val keyEventResult = if (keyDownMS >= longPressTimeout) it.afterKeyLongDown(key, keyDownMS, composing) else it.afterKeyDown(key, composing) handleKeyEventResult(keyEventResult) } ?: super.onKeyUp(keyCode, event) } override fun onKeyLongPress(keyCode: Int, event: KeyEvent?): Boolean { val key = keypad.getKey(keyCode) ?: return super.onKeyLongPress(keyCode, event) return inputMode?.let { val keyEventResult = it.onKeyLongDown(key, composing) handleKeyEventResult(keyEventResult) } ?: super.onKeyLongPress(keyCode, event) } override fun onStartInput(attribute: EditorInfo?, restarting: Boolean) { val inputType = attribute?.inputType?.and(InputType.TYPE_MASK_CLASS) ?: 0 cursorPosition = attribute?.initialSelEnd ?: 0 when (inputType) { InputType.TYPE_CLASS_DATETIME, InputType.TYPE_CLASS_NUMBER, InputType.TYPE_CLASS_PHONE -> enableInputMode(WKT9InputMode.NUMERIC) InputType.TYPE_CLASS_TEXT -> if (lastInputMode == WKT9InputMode.ALPHA) enableInputMode(WKT9InputMode.ALPHA) else enableInputMode(WKT9InputMode.WORD) else -> Log.d(tag, "Mode without input...") } updateInputStatus() super.onStartInput(attribute, restarting) } override fun onUpdateSelection( oldSelStart: Int, oldSelEnd: Int, newSelStart: Int, newSelEnd: Int, candidatesStart: Int, candidatesEnd: Int ) { cursorPosition = newSelEnd super.onUpdateSelection( oldSelStart, oldSelEnd, newSelStart, newSelEnd, candidatesStart, candidatesEnd ) } private fun candidatesToLowerCase() { candidates.forEachIndexed { index, candidate -> candidates[index] = candidate.lowercase() } } private fun candidatesToUpperCase() { candidates.forEachIndexed { index, candidate -> candidates[index] = candidate.uppercase() } } private fun capitalizeCandidates() { candidates.forEachIndexed { index, candidate -> candidates[index] = candidate.lowercase().replaceFirstChar { it.uppercase() } } } private fun clearCandidates() { clearCandidateUI() candidates.clear() candidateIndex = 0 } private fun clearCandidateUI() { val candidatesView = inputView.findViewById(R.id.suggestions) candidatesView.removeAllViews() } private fun commitText(text: CharSequence, start: Int, end: Int): Boolean { return (markComposingRegion(start, end) && composeText(text, 1) && finishComposingText()) } private fun composeText(text: CharSequence, cursorPosition: Int = 1): Boolean { if (!composing) return false return currentInputConnection?.setComposingText(text, cursorPosition) ?: false } private fun deleteText(beforeCursor: Int, afterCursor: Int) { currentInputConnection?.deleteSurroundingText(beforeCursor, afterCursor) updateInputStatus() } // Todo: inputType private fun enableInputMode(mode: WKT9InputMode) { lastInputMode = mode inputMode = when(mode) { WKT9InputMode.ALPHA -> alphaInputMode WKT9InputMode.NUMERIC -> numericInputMode WKT9InputMode.WORD -> wordInputMode } } private fun finishComposingText(): Boolean { return if (composing) { composing = false updateInputStatus() currentInputConnection?.finishComposingText() ?: false } else false } @SuppressLint("DiscouragedApi") private fun getIconResource(): Int { val name = inputMode?.let { val inputMethodManager = getSystemService(INPUT_METHOD_SERVICE) as InputMethodManager val inputMethodSubtype = inputMethodManager.currentInputMethodSubtype it.mode .plus("_") .plus(inputMethodSubtype.languageTag) .plus("_") .plus(inputStatus.toString()) .replace("-", "_") .lowercase() } ?: "wkt9" return resources.getIdentifier(name, "drawable", packageName) } private fun goHome() { with(Intent(Intent.ACTION_MAIN)) { this.addCategory(Intent.CATEGORY_HOME) this.flags = Intent.FLAG_ACTIVITY_NEW_TASK startActivity(this) } } private fun handleComposeTimeout(timeout: Int?) { this.timeout = timeout commitJob?.cancel() if (timeout == null) return commitJob = commitScope.launch { delay(timeout.toLong()) finishComposingText() clearCandidates() } } private fun handleKeyEventResult(res: KeyEventResult): Boolean { if (res.finishComposing) finishComposingText() if (res.startComposing) markComposingRegion() if (!res.codeWord.isNullOrEmpty()) onCodeWordUpdate(res.codeWord, res.timeout) if (!res.candidates.isNullOrEmpty()) onCandidates(res.candidates, res.timeout) if (res.deleteBeforeCursor > 0 || res.deleteAfterCursor > 0) onDelete(res.deleteBeforeCursor, res.deleteAfterCursor) if (res.goHome) goHome() if (res.left) onLeft() if (res.right) onRight() if (res.record) onRecord() if (res.transcribe) onTranscribe() if (res.updateInputStatus) updateInputStatus() if (res.updateWordStatus) onUpdateWordStatus() if (res.focus) onFocus() if (res.switchInputMode != null) onSwitchInputMode(res.switchInputMode) return res.consumed } private fun isSentenceStart(): Boolean { if (cursorPosition == 0) return true val textBeforeCursor = currentInputConnection?.getTextBeforeCursor(10, 0) ?: return false if ( textBeforeCursor.trimEnd().isEmpty() || listOf('.', '!', '?').contains(textBeforeCursor.trimEnd().last())) return true return false } private fun loadCandidates(highLight: Int? = null) { val candidatesView = inputView.findViewById(R.id.suggestions) candidates.forEachIndexed { index, candidate -> val layout = if (index == highLight) R.layout.current_suggestion else R.layout.suggestion val candidateView = layoutInflater.inflate(layout, null) val textView = candidateView.findViewById(R.id.suggestion_text) textView.text = candidate candidatesView.addView(candidateView) } } private fun markComposingRegion(start: Int? = null, end: Int? = null): Boolean { if (composing) return false val composeStart = start ?: cursorPosition val composeEnd = end ?: cursorPosition composing = currentInputConnection?.setComposingRegion(composeStart, composeEnd) ?: false return composing } private fun onCandidates(candidates: List, timeout: Int?) { clearCandidates() candidates.forEach { val candidate = when (inputStatus) { Status.CAP -> it.replaceFirstChar { char -> char.uppercase() } Status.UPPER -> it.uppercase() else -> it } this.candidates.add(candidate) } loadCandidates(candidateIndex) composeText(this.candidates[candidateIndex]) handleComposeTimeout(timeout) } private fun onCodeWordUpdate(codeWord: StringBuilder, timeout: Int?) { clearCandidates() queryJob?.cancel() queryJob = queryScope.launch { val hasCandidates = queryT9Candidates(codeWord, 10) if (!hasCandidates) return@launch loadCandidates(candidateIndex) composeText(candidates[candidateIndex], 1) handleComposeTimeout(timeout) } } private fun onDelete(beforeCursor: Int, afterCursor: Int) { clearCandidates() deleteText(beforeCursor, afterCursor) } private fun onFocus() { requestShowSelf(InputMethodManager.SHOW_IMPLICIT) } private fun onLeft() { if (candidates.isEmpty()) return candidateIndex-- if (candidateIndex < 0) candidateIndex = candidates.count() - 1 clearCandidateUI() loadCandidates(candidateIndex) composeText(candidates[candidateIndex]) handleComposeTimeout(this.timeout) } private fun onRecord() { // The recorder must be busy... if (recorder !== null || !isInputViewShown) return clearCandidates() // Delete possible existing recording recording?.delete() // Toast settings val text = "Recording now.\nRelease the button to start transcribing." val duration = Toast.LENGTH_SHORT // Instantiate recorder and start recording recorder = MediaRecorder(this).also { recording = File.createTempFile("recording.3gp", null, cacheDir) it.setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION) it.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP) it.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB) it.setOutputFile(recording) try { it.prepare() it.start() toast?.cancel() toast = Toast.makeText(this, text, duration).apply { this.show() } } catch (e: Exception) { Log.d(tag, "Failed to start recording", e) } } } private fun onRight() { if (candidates.isEmpty()) return candidateIndex++ if (candidateIndex >= candidates.count()) candidateIndex = 0 clearCandidateUI() loadCandidates(candidateIndex) composeText(candidates[candidateIndex]) handleComposeTimeout(this.timeout) } private fun onSwitchInputMode(mode: WKT9InputMode) { when (mode) { WKT9InputMode.ALPHA -> enableInputMode(WKT9InputMode.ALPHA) WKT9InputMode.NUMERIC -> enableInputMode(WKT9InputMode.NUMERIC) WKT9InputMode.WORD -> enableInputMode(WKT9InputMode.WORD) } clearCandidates() updateInputStatus() } private fun onTranscribe() { val recorder = this.recorder ?: return recorder.stop() recorder.reset() recorder.release() this.recorder = null val text = "Sending recording to speech-to-text server for transcription." val duration = Toast.LENGTH_SHORT toast?.cancel() toast = Toast.makeText(this, text, duration).apply { this.show() } ioJob?.cancel() ioJob = ioScope.launch { try { val transcription = whisper.run(recording!!) commitText(transcription.plus(" "), cursorPosition, cursorPosition) } catch (e: IOException) { Log.d(tag, "A failure occurred in the communication with the speech-to-text server", e) } } } private fun onUpdateWordStatus() { clearCandidateUI() when (inputStatus) { Status.CAP -> { inputStatus = Status.UPPER candidatesToUpperCase() } Status.UPPER -> { inputStatus = Status.LOWER candidatesToLowerCase() } else -> { inputStatus = Status.CAP capitalizeCandidates() } } showStatusIcon(getIconResource()) loadCandidates(candidateIndex) composeText(candidates[candidateIndex]) } private fun updateInputStatus() { inputStatus = inputMode?.status ?: Status.CAP if (inputStatus == Status.CAP && !isSentenceStart()) inputStatus = Status.LOWER showStatusIcon(getIconResource()) } private suspend fun queryT9Candidates(codeWord: StringBuilder, limit: Int = 10): Boolean { val words = wordDao.findCandidates(codeWord.toString(), limit) words.forEach { word -> val candidate = when (inputStatus) { Status.CAP -> word.word.replaceFirstChar { it.uppercase() } Status.UPPER -> word.word.uppercase() else -> word.word } candidates.add(candidate) } return words.isNotEmpty() } }