From 96d0443892bc27bcbde7ba39a0b9b11c0caafffc Mon Sep 17 00:00:00 2001 From: Nehemiah Date: Mon, 13 Nov 2023 11:58:48 -0500 Subject: [PATCH] First release? --- app/src/main/java/net/mezimmah/wkt9/WKT9.kt | 249 ++------------- .../java/net/mezimmah/wkt9/WKT9Interface.kt | 11 +- .../mezimmah/wkt9/candidates/Candidates.kt | 70 ++++ .../wkt9/inputmode/IdleInputHandler.kt | 4 +- .../mezimmah/wkt9/inputmode/InputHandler.kt | 16 +- .../wkt9/inputmode/InputHandlerInterface.kt | 4 +- .../mezimmah/wkt9/inputmode/InputManager.kt | 24 +- .../wkt9/inputmode/LetterInputHandler.kt | 299 ++++++++++-------- .../wkt9/inputmode/WordInputHandler.kt | 185 ++++++----- .../java/net/mezimmah/wkt9/voice/Whisper.kt | 115 ++++++- app/src/main/res/drawable/mic.xml | 5 + app/src/main/res/layout/message.xml | 29 ++ app/src/main/res/layout/suggestions.xml | 70 +++- 13 files changed, 621 insertions(+), 460 deletions(-) create mode 100644 app/src/main/java/net/mezimmah/wkt9/candidates/Candidates.kt create mode 100644 app/src/main/res/drawable/mic.xml create mode 100644 app/src/main/res/layout/message.xml diff --git a/app/src/main/java/net/mezimmah/wkt9/WKT9.kt b/app/src/main/java/net/mezimmah/wkt9/WKT9.kt index 04f809b..b2922d6 100644 --- a/app/src/main/java/net/mezimmah/wkt9/WKT9.kt +++ b/app/src/main/java/net/mezimmah/wkt9/WKT9.kt @@ -11,23 +11,13 @@ import android.view.View import android.view.inputmethod.EditorInfo import android.view.inputmethod.ExtractedTextRequest import android.view.inputmethod.InputMethodManager -import android.widget.LinearLayout -import android.widget.TextView -import android.widget.Toast -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch +import net.mezimmah.wkt9.candidates.Candidates import net.mezimmah.wkt9.inputmode.InputManager import net.mezimmah.wkt9.inputmode.InputMode import net.mezimmah.wkt9.keypad.Event import net.mezimmah.wkt9.keypad.Key import net.mezimmah.wkt9.keypad.KeyEventStat import net.mezimmah.wkt9.voice.Whisper -import okio.IOException -import java.io.File class WKT9: WKT9Interface, InputMethodService() { private val tag = "WKT9" @@ -38,37 +28,19 @@ class WKT9: WKT9Interface, InputMethodService() { private var inputView: View? = null private var composing: Boolean = false - private var lastComposedText: CharSequence = "" private var cursorPosition: Int = 0 private val keyDownStats = KeyEventStat(0, 0) private val keyUpStats = KeyEventStat(0, 0) // Whisper - private val whisper: Whisper = Whisper() + private lateinit var whisper: Whisper private var recorder: MediaRecorder? = null - private var recording: File? = null - private var toast: Toast? = null + private lateinit var candidates: Candidates - private val ioScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - private var ioJob: Job? = null - - private val commitScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) - private var commitJob: Job? = null - - override fun onCandidates( - candidates: List, - finishComposing: Boolean, - current: Int?, - timeout: Long?, - start: Int?, - end: Int?, - notifyOnChange: Boolean - ) { - if (finishComposing) finishComposingText() - - loadCandidates(candidates, current, timeout, start, end, notifyOnChange) + override fun onCandidates(candidates: List, current: Int?) { + this.candidates.load(candidates, current) } override fun onCancelCompose() { @@ -76,10 +48,24 @@ class WKT9: WKT9Interface, InputMethodService() { } override fun onClearCandidates() { - clearCandidates() + candidates.clear() + } + + override fun onCompose(text: String, rangeStart: Int, rangeEnd: Int) { + val selectionEnd = rangeStart + text.length + + currentInputConnection?.run { + beginBatchEdit() + setComposingRegion(rangeStart, rangeEnd) + commitText(text, 1) + setSelection(rangeStart, selectionEnd) + endBatchEdit() + } } override fun onCreate() { + Log.d(tag, "Starting WKT9") + inputManager = InputManager(this) super.onCreate() @@ -87,7 +73,10 @@ class WKT9: WKT9Interface, InputMethodService() { @SuppressLint("InflateParams") override fun onCreateInputView(): View? { - inputView = layoutInflater.inflate(R.layout.suggestions, null) + inputView = layoutInflater.inflate(R.layout.suggestions, null).also { + candidates = Candidates(this, inputManager, it) + whisper = Whisper(this, inputManager, it) + } return inputView } @@ -98,14 +87,16 @@ class WKT9: WKT9Interface, InputMethodService() { deleteText(beforeCursor, afterCursor) } - override fun onFinishComposing() { - finishComposingText() + override fun onFinishComposing(cursorPosition: Int) { + currentInputConnection?.run { + setSelection(cursorPosition, cursorPosition) + } } override fun onFinishInputView(finishingInput: Boolean) { super.onFinishInputView(finishingInput) - clearCandidates() + onClearCandidates() } override fun onGetTextBeforeCursor(n: Int): CharSequence? { @@ -243,9 +234,6 @@ class WKT9: WKT9Interface, InputMethodService() { } override fun onStartInput(editorInfo: EditorInfo?, restarting: Boolean) { - ioJob?.cancel() // Cancel possible transcription job - toast?.cancel() - if (editorInfo == null) return inputManager.selectHandler(editorInfo) @@ -259,13 +247,7 @@ class WKT9: WKT9Interface, InputMethodService() { candidatesStart: Int, candidatesEnd: Int ) { - inputManager.handler?.run { - if (candidatesStart != candidatesEnd) { - onComposeText(lastComposedText, candidatesStart, candidatesEnd) - } - - onUpdateCursorPosition(newSelEnd) - } + inputManager.handler?.onUpdateCursorPosition(newSelEnd) super.onUpdateSelection( oldSelStart, @@ -311,37 +293,6 @@ class WKT9: WKT9Interface, InputMethodService() { } } - private fun clearCandidates() { - val candidatesView = inputView?.findViewById(R.id.suggestions) ?: return - - candidatesView.removeAllViews() - } - - private fun markComposingRegion(start: Int? = null, end: Int? = null) { - this.composing = currentInputConnection?.run { - val composeStart = start ?: cursorPosition - val composeEnd = end ?: cursorPosition - - setComposingRegion(composeStart, composeEnd) - } ?: false - } - - private fun commitText(text: CharSequence) { - val insertTextStart = cursorPosition - val insertTextEnd = cursorPosition + text.length - - currentInputConnection?.commitText(text, 1) - inputManager.handler?.onInsertText(text, insertTextStart, insertTextEnd) - } - - private fun composeText(text: CharSequence, cursorPosition: Int = 1) { - if (!composing) return - - currentInputConnection?.let { - if (it.setComposingText(text, cursorPosition)) lastComposedText = text - } - } - private fun deleteText(beforeCursor: Int, afterCursor: Int) { currentInputConnection?.run { deleteSurroundingText(beforeCursor, afterCursor) @@ -360,148 +311,16 @@ class WKT9: WKT9Interface, InputMethodService() { composing = false } - private fun getCandidateIndex(candidate: View): Int? { - val candidatesView = inputView?.findViewById(R.id.suggestions) ?: return null - - for (i in 0 until candidatesView.childCount) { - val child: View = candidatesView.getChildAt(i) - - if (candidate == child) return i - } - - return null - } - - private fun loadCandidates( - candidates: List, - current: Int?, - timeout: Long?, - start: Int?, - end: Int?, - notifyOnChange: Boolean = false - ) { - val candidatesView = inputView?.findViewById(R.id.suggestions) ?: return - - clearCandidates() - commitJob?.cancel() - - candidates.forEachIndexed { index, candidate -> - val layout = if (index == current) R.layout.current_suggestion else R.layout.suggestion - val candidateView = layoutInflater.inflate(layout, null) - val textView = candidateView.findViewById(R.id.suggestion_text) - - textView.text = candidate - - candidateView.setOnClickListener { view -> - getCandidateIndex(view)?.let { index -> - loadCandidates(candidates, index, timeout, start, end, notifyOnChange) - } - } - - candidateView.setOnLongClickListener { - val view = it as LinearLayout - val suggestion = view.findViewById(R.id.suggestion_text) - val text = suggestion.text - - inputManager.handler?.onLongClickCandidate(text.toString()) - - true - } - - if (index == current) { - if (!composing) markComposingRegion(start, end) - - composeText(candidate) - - if (notifyOnChange) { - inputManager.handler?.onCandidateSelected(candidate) - } - } - - candidatesView.addView(candidateView) - } - - if (!isInputViewShown) requestShowSelf(InputMethodManager.SHOW_IMPLICIT) - if (timeout == null) return - - commitJob = commitScope.launch { - delay(timeout.toLong()) - resetKeyStats() - finishComposingText() - } - } - - @Suppress("DEPRECATION") private fun record() { // The recorder must be busy... - if (recorder !== null) return - - clearCandidates() + if (recorder != null) return + requestShowSelf(InputMethodManager.SHOW_IMPLICIT) - // Delete possible existing recording - recording?.delete() - - // Toast settings - val text = "Recording now.\nRelease the button to start transcribing." - val duration = Toast.LENGTH_SHORT - - // Instantiate recorder and start recording - recorder = MediaRecorder().also { - recording = File.createTempFile("recording.3gp", null, cacheDir) - - it.setAudioSource(MediaRecorder.AudioSource.MIC) - it.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP) - it.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB) - it.setOutputFile(recording) - - try { - it.prepare() - it.start() - - toast?.cancel() - toast = Toast.makeText(this, text, duration).apply { - this.show() - } - } catch (e: Exception) { - Log.d(tag, "Failed to start recording", e) - } - } - } - - private fun resetKeyStats() { - keyDownStats.keyCode = 0 - keyDownStats.repeats = 0 - keyUpStats.keyCode = 0 - keyUpStats.repeats = 0 + whisper.record() } private fun transcribe() { - val recorder = this.recorder ?: return - - recorder.stop() - recorder.reset() - recorder.release() - - this.recorder = null - - val text = "Sending recording to speech-to-text server for transcription." - val duration = Toast.LENGTH_SHORT - - toast?.cancel() - toast = Toast.makeText(this, text, duration).apply { - this.show() - } - - ioJob?.cancel() - ioJob = ioScope.launch { - try { - val transcription = whisper.run(recording!!) - - commitText(transcription.plus(' ')) - } catch (e: IOException) { - Log.d(tag, "A failure occurred in the communication with the speech-to-text server", e) - } - } + whisper.transcribe() } } \ No newline at end of file diff --git a/app/src/main/java/net/mezimmah/wkt9/WKT9Interface.kt b/app/src/main/java/net/mezimmah/wkt9/WKT9Interface.kt index 3c1c9fe..45f4a0e 100644 --- a/app/src/main/java/net/mezimmah/wkt9/WKT9Interface.kt +++ b/app/src/main/java/net/mezimmah/wkt9/WKT9Interface.kt @@ -11,21 +11,18 @@ interface WKT9Interface { fun onCandidates( candidates: List, - finishComposing: Boolean = false, - current: Int? = 0, - timeout: Long? = null, - start: Int? = null, - end: Int? = null, - notifyOnChange: Boolean = false + current: Int? = 0 ) fun onCancelCompose() fun onClearCandidates() + fun onCompose(text: String, rangeStart: Int, rangeEnd: Int) + fun onDeleteText(beforeCursor: Int = 0, afterCursor: Int = 0, finishComposing: Boolean) - fun onFinishComposing() + fun onFinishComposing(cursorPosition: Int) fun onGetText(): CharSequence? diff --git a/app/src/main/java/net/mezimmah/wkt9/candidates/Candidates.kt b/app/src/main/java/net/mezimmah/wkt9/candidates/Candidates.kt new file mode 100644 index 0000000..a5a0339 --- /dev/null +++ b/app/src/main/java/net/mezimmah/wkt9/candidates/Candidates.kt @@ -0,0 +1,70 @@ +package net.mezimmah.wkt9.candidates + +import android.view.View +import android.view.inputmethod.InputMethodManager +import android.widget.LinearLayout +import android.widget.TextView +import net.mezimmah.wkt9.R +import net.mezimmah.wkt9.WKT9 +import net.mezimmah.wkt9.inputmode.InputManager + +class Candidates( + private val context: WKT9, + private val inputManager: InputManager, + + ui: View +) { + private val candidatesView: LinearLayout = ui.findViewById(R.id.suggestions) + + fun clear() { + candidatesView.removeAllViews() + } + + fun load(candidates: List, current: Int?) { + clear() + + candidates.forEachIndexed { index, candidate -> + val layout = if (index == current) R.layout.current_suggestion else R.layout.suggestion + val candidateView = context.layoutInflater.inflate(layout, null) + val textView = candidateView.findViewById(R.id.suggestion_text) + + textView.text = candidate + + candidateView.setOnClickListener { view -> + getIndex(view)?.let { index -> + load(candidates, index) + } + } + + candidateView.setOnLongClickListener { + val view = it as LinearLayout + val suggestion = view.findViewById(R.id.suggestion_text) + val text = suggestion.text + + inputManager.handler?.onLongClickCandidate(text.toString()) + + true + } + + if (index == current) { + inputManager.handler?.onCandidateSelected(index) + } + + candidatesView.addView(candidateView) + } + + if (!context.isInputViewShown) { + context.requestShowSelf(InputMethodManager.SHOW_IMPLICIT) + } + } + + private fun getIndex(candidate: View): Int? { + for (i in 0 until candidatesView.childCount) { + val child: View = candidatesView.getChildAt(i) + + if (candidate == child) return i + } + + return null + } +} \ No newline at end of file diff --git a/app/src/main/java/net/mezimmah/wkt9/inputmode/IdleInputHandler.kt b/app/src/main/java/net/mezimmah/wkt9/inputmode/IdleInputHandler.kt index dcf4fdb..41cc0f1 100644 --- a/app/src/main/java/net/mezimmah/wkt9/inputmode/IdleInputHandler.kt +++ b/app/src/main/java/net/mezimmah/wkt9/inputmode/IdleInputHandler.kt @@ -20,8 +20,8 @@ class IdleInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, override fun onRunCommand(command: Command, key: Key, event: KeyEvent, stats: KeyEventStat) { when (command) { Command.DIAL -> dial() - Command.CAMERA -> triggerKeyEvent(KeyEvent.KEYCODE_CAMERA, false) - Command.NUMBER -> triggerOriginalKeyEvent(key, false) + Command.CAMERA -> triggerKeyEvent(KeyEvent.KEYCODE_CAMERA) + Command.NUMBER -> triggerOriginalKeyEvent(key) else -> Log.d(tag, "Command not implemented: $command") } } diff --git a/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandler.kt b/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandler.kt index ffa63a8..a8a9419 100644 --- a/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandler.kt +++ b/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandler.kt @@ -1,7 +1,6 @@ package net.mezimmah.wkt9.inputmode import android.text.InputType -import android.util.Log import android.view.KeyEvent import net.mezimmah.wkt9.WKT9 import net.mezimmah.wkt9.WKT9Interface @@ -28,9 +27,7 @@ open class InputHandler( override var cursorPosition: Int = 0 protected set - override fun onCandidateSelected(candidate: String) { - Log.d(tag, "A candidate has been selected: $candidate") - } + override fun onCandidateSelected(index: Int) {} override fun onCommitText() {} @@ -38,7 +35,7 @@ open class InputHandler( override fun onFinish() {} - override fun onInsertText(text: CharSequence, insertTextStart: Int, insertTextEnd: Int) {} + override fun onInsertText(text: CharSequence) {} override fun onLongClickCandidate(text: String) {} @@ -76,7 +73,6 @@ open class InputHandler( wkt9.onCandidates( candidates = candidates, - finishComposing = stats.repeats == 0, current = stats.repeats % candidates.count() ) } @@ -107,17 +103,15 @@ open class InputHandler( return null } - protected fun triggerKeyEvent(keyCode: Int, finishComposing: Boolean) { + protected fun triggerKeyEvent(keyCode: Int) { val down = KeyEvent(KeyEvent.ACTION_DOWN, keyCode) val up = KeyEvent(KeyEvent.ACTION_UP, keyCode) - if (finishComposing) wkt9.onFinishComposing() - wkt9.onTriggerKeyEvent(down) wkt9.onTriggerKeyEvent(up) } - protected fun triggerOriginalKeyEvent(key: Key, finishComposing: Boolean) { - triggerKeyEvent(key.keyCode, finishComposing) + protected fun triggerOriginalKeyEvent(key: Key) { + triggerKeyEvent(key.keyCode) } } \ No newline at end of file diff --git a/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandlerInterface.kt b/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandlerInterface.kt index 1cdd6b5..cd1a69c 100644 --- a/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandlerInterface.kt +++ b/app/src/main/java/net/mezimmah/wkt9/inputmode/InputHandlerInterface.kt @@ -14,7 +14,7 @@ interface InputHandlerInterface { val capMode: Int? val cursorPosition: Int? - fun onCandidateSelected(candidate: String) + fun onCandidateSelected(index: Int) fun onComposeText(text: CharSequence, composingTextStart: Int, composingTextEnd: Int) @@ -22,7 +22,7 @@ interface InputHandlerInterface { fun onFinish() - fun onInsertText(text: CharSequence, insertTextStart: Int, insertTextEnd: Int) + fun onInsertText(text: CharSequence) fun onLongClickCandidate(text: String) diff --git a/app/src/main/java/net/mezimmah/wkt9/inputmode/InputManager.kt b/app/src/main/java/net/mezimmah/wkt9/inputmode/InputManager.kt index ee06626..a73c512 100644 --- a/app/src/main/java/net/mezimmah/wkt9/inputmode/InputManager.kt +++ b/app/src/main/java/net/mezimmah/wkt9/inputmode/InputManager.kt @@ -1,6 +1,7 @@ package net.mezimmah.wkt9.inputmode import android.text.InputType +import android.util.Log import android.view.inputmethod.EditorInfo import net.mezimmah.wkt9.R import net.mezimmah.wkt9.WKT9 @@ -51,7 +52,7 @@ class InputManager(val context: WKT9) { if (override != null) return switchToHandler(override, editor.initialSelEnd) - val handler = if (numericClasses.contains(typeClass)) { + val mode = if (numericClasses.contains(typeClass)) { InputMode.Number } else if (typeClass == InputType.TYPE_CLASS_TEXT) { if (letterVariations.contains(typeVariation) || mode == InputMode.Letter) { @@ -63,21 +64,28 @@ class InputManager(val context: WKT9) { InputMode.Idle } - switchToHandler(handler, editor.initialSelEnd) + switchToHandler(mode, editor.initialSelEnd) } fun switchToHandler(inputMode: InputMode, cursorPosition: Int) { - this.handler?.onFinish() - this.mode = inputMode - this.handler = when (inputMode) { + val lastHandler = this.handler + val cursor: Int = lastHandler?.cursorPosition ?: cursorPosition + val newHandler = when (inputMode) { InputMode.Word -> wordInputHandler InputMode.Letter -> letterInputHandler InputMode.Number -> numberInputHandler else -> idleInputHandler - }.apply { - onStart(typeClass, typeVariation, typeFlags) - onUpdateCursorPosition(cursorPosition) } + + newHandler.apply { + onStart(typeClass, typeVariation, typeFlags) + onUpdateCursorPosition(cursor) + } + + lastHandler?.onFinish() + + this.mode = inputMode + this.handler = newHandler } private fun selectOverride(packageName: String): InputMode? { diff --git a/app/src/main/java/net/mezimmah/wkt9/inputmode/LetterInputHandler.kt b/app/src/main/java/net/mezimmah/wkt9/inputmode/LetterInputHandler.kt index a51d93e..4a236b5 100644 --- a/app/src/main/java/net/mezimmah/wkt9/inputmode/LetterInputHandler.kt +++ b/app/src/main/java/net/mezimmah/wkt9/inputmode/LetterInputHandler.kt @@ -7,11 +7,12 @@ import android.view.KeyEvent import android.view.textservice.SentenceSuggestionsInfo import android.view.textservice.SpellCheckerSession import android.view.textservice.SuggestionsInfo -import android.view.textservice.TextInfo import android.view.textservice.TextServicesManager import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay import kotlinx.coroutines.launch import net.mezimmah.wkt9.R import net.mezimmah.wkt9.WKT9 @@ -25,20 +26,25 @@ import net.mezimmah.wkt9.keypad.Key import net.mezimmah.wkt9.keypad.KeyEventStat import net.mezimmah.wkt9.keypad.KeyLayout import java.util.Locale -import kotlin.text.StringBuilder class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSession.SpellCheckerSessionListener, InputHandler(wkt9, context) { private var db: AppDatabase private var wordDao: WordDao private var locale: Locale? = null private var spellCheckerSession: SpellCheckerSession? = null + + private val candidates = mutableListOf() + private var candidateIndex: Int? = null + private var composeRangeStart: Int? = null + private var composeRangeEnd: Int? = null + private var sentenceStart: Boolean = false private var wordStart: Boolean = false - private var selectionStart: Int = 0 private val queryScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + private val timeoutScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + private var timeoutJob: Job? = null - private val composing = StringBuilder() private val content = StringBuilder() init { @@ -66,39 +72,42 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio updateIcon() } - override fun onCandidateSelected(candidate: String) { - wkt9.onFinishComposing() - } + override fun finalizeWordOrSentence(stats: KeyEventStat) { + if (stats.repeats == 0) { + timeoutJob?.cancel() - override fun onCommitText() { - composing.clear() - wkt9.onClearCandidates() - - val info = getCursorPositionInfo(content) - - if (info.startSentence || info.startWord) return - - val last = content.split("\\s".toRegex()).last() - - if (last.length > 2) getSuggestions(last) - } - - override fun onComposeText(text: CharSequence, composingTextStart: Int, composingTextEnd: Int) { - val info = getCursorPositionInfo(text) - val lastWord = content.split("\\s+".toRegex()).last() - - if (lastWord.isNotEmpty() && (info.startSentence || info.startWord)) { - storeWord(lastWord) + clearCandidates(true) + storeLastWord() } - content.replace(composingTextStart, composingTextEnd, text.toString()) - composing.replace(0, composing.length, text.toString()) + candidates.addAll(listOf(" ", ". ", "? ", "! ", ", ", ": ", "; ")) + wkt9.onCandidates(candidates, stats.repeats % candidates.count()) + } + + override fun onCandidateSelected(index: Int) { + val candidate = candidates[index] + val rangeStart = composeRangeStart ?: cursorPosition + val rangeEnd = composeRangeEnd ?: cursorPosition + + content.replace(rangeStart, rangeEnd, candidate) + + candidateIndex = index + composeRangeStart = rangeStart + composeRangeEnd = rangeStart + candidate.length + + wkt9.onCompose(candidate, rangeStart, rangeEnd) } override fun onFinish() { - super.onFinish() + timeoutJob?.cancel() - wkt9.onFinishComposing() + clearCandidates(true) + + cursorPosition = 0 + sentenceStart = false + wordStart = false + + content.clear() } override fun onGetSuggestions(results: Array?) { @@ -106,32 +115,15 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio } override fun onGetSentenceSuggestions(results: Array?) { - val candidates = mutableListOf() - - results?.map { - val suggestions = it.getSuggestionsInfoAt(0) - - for (index in 0 until suggestions.suggestionsCount) { - val suggestion = suggestions.getSuggestionAt(index) - - candidates.add(suggestion) - } - } - - if (candidates.isEmpty()) return - - wkt9.onCandidates( - candidates = candidates, - current = null, - start = selectionStart, - end = cursorPosition, - notifyOnChange = true - ) + TODO("Not yet implemented") } - // This is for the text that should be committed without 'handling'. - override fun onInsertText(text: CharSequence, insertTextStart: Int, insertTextEnd: Int) { - content.replace(insertTextStart, insertTextEnd, text.toString()) + override fun onInsertText(text: CharSequence) { + clearCandidates(true) + + content.replace(cursorPosition, cursorPosition, text.toString()) + wkt9.onCompose(text.toString(), cursorPosition, cursorPosition) + wkt9.onFinishComposing(cursorPosition + text.length) } override fun onRunCommand(command: Command, key: Key, event: KeyEvent, stats: KeyEventStat) { @@ -141,7 +133,7 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio Command.DELETE -> delete() Command.INPUT_MODE -> inputMode(key) Command.MOVE_CURSOR -> moveCursor() - Command.NUMBER -> triggerOriginalKeyEvent(key, true) + Command.NUMBER -> triggerOriginalKeyEvent(key) Command.RECORD -> wkt9.onRecord(true) Command.SPACE -> finalizeWordOrSentence(stats) Command.TRANSCRIBE -> wkt9.onTranscribe() @@ -149,6 +141,21 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio } } + override fun onUpdateCursorPosition(cursorPosition: Int) { + super.onUpdateCursorPosition(cursorPosition) + + try { + val info = getCursorPositionInfo(content.substring(0, cursorPosition)) + + sentenceStart = info.startSentence + wordStart = info.startWord + } catch (e: Exception) { + Log.d(tag, "Cursor position out of range.\nContent: $content\nCursor position: $cursorPosition", e) + } + + updateIcon() + } + override fun onStart(typeClass: Int, typeVariations: Int, typeFlags: Int) { capMode = getDefaultCapMode(typeFlags) @@ -158,72 +165,78 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio } ?: content.clear() } - override fun onUpdateCursorPosition(cursorPosition: Int) { - super.onUpdateCursorPosition(cursorPosition) + private fun composeCharacter(key: Key, stats: KeyEventStat) { + val layout = KeyLayout.en_US[key] ?: return + val capitalize = Capitalize(capMode) - if (cursorPosition > content.length) return + if (stats.repeats == 0) clearCandidates(true) - var info: CursorPositionInfo - - if (cursorPosition == 0) { - info = CursorPositionInfo( - startSentence = true, - startWord = true - ) - } else if (composing.isNotEmpty()) { - info = getCursorPositionInfo(composing) - - if (!info.startSentence && !info.startWord) { - info = getCursorPositionInfo(content.substring(0, cursorPosition - composing.length)) - } - } else { - info = getCursorPositionInfo(content.substring(0, cursorPosition)) + layout.forEach { + candidates.add(capitalize.character(it, sentenceStart, wordStart)) } - sentenceStart = info.startSentence - wordStart = info.startWord - - updateIcon() - } - - private fun getSuggestions(text: String){ - val words = arrayOf( - TextInfo( - text.plus("#"), // Add hash to string to get magic performance - 0, - text.length + 1, // We added the hash, remember - 0, - 0 - ) + wkt9.onCandidates( + candidates = candidates, + current = stats.repeats % candidates.count() ) - selectionStart = cursorPosition - text.length + timeoutJob?.cancel() + timeoutJob = timeoutScope.launch { + delay(400) + clearCandidates(true) +// getSuggestions() + } + } - spellCheckerSession?.getSentenceSuggestions(words, 15) + private fun clearCandidates(finishComposing: Boolean) { + if (finishComposing && candidateIndex != null) { + wkt9.onFinishComposing(cursorPosition) + } + + candidateIndex = null + composeRangeStart = null + composeRangeEnd = null + + candidates.clear() + wkt9.onClearCandidates() + } + + private fun delete() { + if (candidateIndex != null) undoCandidate() + else if (content.isNotEmpty()) { + content.deleteAt(content.length - 1) + wkt9.onDeleteText(1, 0, true) + } + } + + private fun getLastWord(): String { + val beforeCursor = content.substring(0, cursorPosition) + val words = beforeCursor.split("\\s+".toRegex()) + + return words.last() + } + + private fun inputMode(key: Key) { + if (key == Key.B4) wkt9.onSwitchInputHandler(InputMode.Word) + else wkt9.onSwitchInputHandler(InputMode.Number) } private fun moveCursor() { - if (composing.isNotEmpty()) wkt9.onFinishComposing() + timeoutJob?.cancel() + clearCandidates(true) } - private fun storeWord(text: String) { + private fun storeLastWord() { + val lastWord = getLastWord() // We're not storing single char words... - if (text.length < 2) return - - val words = text.trim().split("\\s+".toRegex()) - - if (words.count() > 1) { - words.forEach { storeWord(it) } - - return - } + if (lastWord.length < 2) return try { - val codeword = keypad.getCodeForWord(text) + val codeword = keypad.getCodeForWord(lastWord) val word = Word( - word = text, + word = lastWord, code = codeword, - length = text.length, + length = lastWord.length, weight = 1, locale = "en_US" ) @@ -236,38 +249,17 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio } } - private fun composeCharacter(key: Key, stats: KeyEventStat) { - val layout = KeyLayout.en_US[key] ?: return - val capitalize = Capitalize(capMode) - val candidates = mutableListOf() + private fun undoCandidate() { + val rangeStart = composeRangeStart ?: cursorPosition + val rangeEnd = composeRangeEnd ?: cursorPosition - if (stats.repeats == 0 && composing.isNotEmpty()) wkt9.onFinishComposing() + // Remove text codeword produced from editor + wkt9.onCompose("", rangeStart, rangeEnd) - layout.forEach { - candidates.add(capitalize.character(it, sentenceStart, wordStart)) - } + // Remove text codeword produced from content + content.deleteRange(rangeStart, rangeEnd) - wkt9.onCandidates( - candidates = candidates, - current = stats.repeats % candidates.count(), - timeout = 400L - ) - } - - private fun delete() { - if (composing.isNotEmpty()) { - wkt9.onCancelCompose() - content.delete(cursorPosition - composing.length, cursorPosition) - composing.clear() - } else if (content.isNotEmpty()) { - content.deleteAt(content.length - 1) - wkt9.onDeleteText(1, 0, true) - } - } - - private fun inputMode(key: Key) { - if (key == Key.B4) wkt9.onSwitchInputHandler(InputMode.Word) - else wkt9.onSwitchInputHandler(InputMode.Number) + clearCandidates(false) } private fun updateIcon() { @@ -280,4 +272,43 @@ class LetterInputHandler(wkt9: WKT9Interface, context: WKT9): SpellCheckerSessio wkt9.onUpdateStatusIcon(icon) } -} \ No newline at end of file +} +// +// override fun onGetSentenceSuggestions(results: Array?) { +// results?.map { +// val suggestions = it.getSuggestionsInfoAt(0) +// +// for (index in 0 until suggestions.suggestionsCount) { +// val suggestion = suggestions.getSuggestionAt(index) +// +// candidates.add(suggestion) +// } +// } +// +// if (candidates.isEmpty()) return +// +// candidateSource = CandidateSource.Dictionary +// +// wkt9.onCandidates( +// candidates = candidates, +// current = null +// ) +// } + +// private fun getSuggestions() { +// val lastWord = getLastWord() +// +// if (lastWord.length < 3) return +// +// val words = arrayOf( +// TextInfo( +// lastWord.plus("#"), // Add hash to string to get magic performance +// 0, +// lastWord.length + 1, // We added the hash, remember +// 0, +// 0 +// ) +// ) +// +// spellCheckerSession?.getSentenceSuggestions(words, 15) +// } \ No newline at end of file diff --git a/app/src/main/java/net/mezimmah/wkt9/inputmode/WordInputHandler.kt b/app/src/main/java/net/mezimmah/wkt9/inputmode/WordInputHandler.kt index 9ebab99..73d60b9 100644 --- a/app/src/main/java/net/mezimmah/wkt9/inputmode/WordInputHandler.kt +++ b/app/src/main/java/net/mezimmah/wkt9/inputmode/WordInputHandler.kt @@ -23,7 +23,10 @@ import net.mezimmah.wkt9.t9.T9 class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, context) { private val content = StringBuilder() private val codeword = StringBuilder() - private val composing = StringBuilder() + private val candidates = mutableListOf() + private var candidateIndex: Int? = null + private var composeRangeStart: Int? = null + private var composeRangeEnd: Int? = null private var db: AppDatabase private var wordDao: WordDao @@ -63,24 +66,45 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, if (codeword.isNotEmpty()) handleCodewordChange(codeword) } - override fun onCommitText() { - if (codeword.isNotEmpty()) increaseWordWeight(composing.toString()) + override fun finalizeWordOrSentence(stats: KeyEventStat) { + if (codeword.isNotEmpty()) commit() - clearCodeword() + candidates.addAll(listOf(" ", ". ", "? ", "! ", ", ", ": ", "; ")) + wkt9.onCandidates(candidates, stats.repeats % candidates.count()) } - override fun onComposeText(text: CharSequence, composingTextStart: Int, composingTextEnd: Int) { - content.replace(composingTextStart, composingTextEnd, text.toString()) - composing.replace(0, composing.length, text.toString()) + override fun onCandidateSelected(index: Int) { + val candidate = candidates[index] + val rangeStart = composeRangeStart ?: cursorPosition + val rangeEnd = composeRangeEnd ?: cursorPosition + + content.replace(rangeStart, rangeEnd, candidate) + + candidateIndex = index + composeRangeStart = rangeStart + composeRangeEnd = rangeStart + candidate.length + + wkt9.onCompose(candidate, rangeStart, rangeEnd) } - // This is for the text that should be committed without 'handling'. - override fun onInsertText(text: CharSequence, insertTextStart: Int, insertTextEnd: Int) { - content.replace(insertTextStart, insertTextEnd, text.toString()) + override fun onInsertText(text: CharSequence) { + commit() + + content.replace(cursorPosition, cursorPosition, text.toString()) + wkt9.onCompose(text.toString(), cursorPosition, cursorPosition) + wkt9.onFinishComposing(cursorPosition + text.length) } override fun onFinish() { - if (composing.isNotEmpty()) wkt9.onFinishComposing() + queryJob?.cancel() + + commit() + + cursorPosition = 0 + sentenceStart = false + wordStart = false + + content.clear() } override fun onLongClickCandidate(text: String) { @@ -98,8 +122,8 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, Command.DELETE -> delete() Command.INPUT_MODE -> inputMode(key) Command.MOVE_CURSOR -> moveCursor() - Command.NUMBER -> triggerOriginalKeyEvent(key, true) - Command.RECORD -> wkt9.onRecord(true) + Command.NUMBER -> triggerOriginalKeyEvent(key) + Command.RECORD -> record() Command.SPACE -> finalizeWordOrSentence(stats) Command.TRANSCRIBE -> wkt9.onTranscribe() else -> Log.d(tag, "Command not implemented: $command") @@ -107,10 +131,6 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, } override fun onStart(typeClass: Int, typeVariations: Int, typeFlags: Int) { - codeword.clear() - content.clear() - composing.clear() - capMode = getDefaultCapMode(typeFlags) // Get current editor content on start @@ -122,28 +142,15 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, override fun onUpdateCursorPosition(cursorPosition: Int) { super.onUpdateCursorPosition(cursorPosition) - if (cursorPosition > content.length) return + try { + val info = getCursorPositionInfo(content.substring(0, cursorPosition)) - var info: CursorPositionInfo - - if (cursorPosition == 0) { - info = CursorPositionInfo( - startSentence = true, - startWord = true - ) - } else if (composing.isNotEmpty()) { - info = getCursorPositionInfo(composing) - - if (!info.startSentence && !info.startWord) { - info = getCursorPositionInfo(content.substring(0, cursorPosition - composing.length)) - } - } else { - info = getCursorPositionInfo(content.substring(0, cursorPosition)) + sentenceStart = info.startSentence + wordStart = info.startWord + } catch (e: Exception) { + Log.d(tag, "Cursor position out of range", e) } - sentenceStart = info.startSentence - wordStart = info.startWord - updateIcon() } @@ -151,44 +158,55 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, // Don't build fruitless codeword if (staleCodeword) return - val code = KeyLayout.numeric[key] + if (codeword.isEmpty() && candidateIndex != null) clearCandidates(true) - /** - * This happens when some other method than buildCodeword composed text. With, for example, - * finalizeWordOrSentence. - */ - if (codeword.isEmpty()) wkt9.onFinishComposing() + val code = KeyLayout.numeric[key] codeword.append(code) handleCodewordChange(codeword) } - private fun clearCodeword() { - codeword.clear() - composing.clear() + private fun clearCandidates(finishComposing: Boolean) { + candidateIndex = null + composeRangeStart = null + composeRangeEnd = null + candidates.clear() + + wkt9.onClearCandidates() + + if (finishComposing) wkt9.onFinishComposing(cursorPosition) + } + + private fun clearCodeword(increaseWordWeight: Boolean) { staleCodeword = false + + codeword.clear() + + if (!increaseWordWeight) return + + candidateIndex?.let { + val candidate = candidates[it] + + if (candidate.isNotEmpty()) increaseWordWeight(candidate) + } + } + + private fun commit() { + if (codeword.isNotEmpty()) clearCodeword(true) + if (candidates.isNotEmpty()) clearCandidates(true) } private fun delete() { staleCodeword = false - if (codeword.length > 1) { - codeword.deleteAt(codeword.length - 1) - - handleCodewordChange(codeword) - } else if (codeword.isNotEmpty()) { - clearCodeword() - wkt9.onCancelCompose() + if (codeword.length > 1) reduceCodeword() + else if (codeword.isNotEmpty()) undoCodeword() + else if (candidateIndex != null) undoCandidate() + else if (content.isNotEmpty()) { content.deleteAt(content.length - 1) - } else if (composing.isNotEmpty()) { - wkt9.onCancelCompose() - content.delete(cursorPosition - composing.length, cursorPosition) - composing.clear() - } else if (content.isNotEmpty()) { - content.deleteAt(content.length - 1) - wkt9.onDeleteText(1, 0, true) + wkt9.onDeleteText(1, 0, false) } } @@ -196,14 +214,10 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, queryJob?.cancel() queryJob = queryScope.launch { - val candidates = queryT9Candidates(codeword, 25) + queryT9Candidates(codeword, 25) - if (candidates.isEmpty()) { - staleCodeword = true - - queryJob?.cancel() - wkt9.onClearCandidates() - } else { + if (candidates.isEmpty()) staleCodeword = true + else { wkt9.onCandidates( candidates = candidates, current = 0 @@ -224,10 +238,18 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, } private fun moveCursor() { - if (composing.isEmpty()) return + commit() + } - wkt9.onFinishComposing() - wkt9.onClearCandidates() + private fun record() { + commit() + wkt9.onRecord(true) + } + + private fun reduceCodeword() { + codeword.deleteAt(codeword.length - 1) + + handleCodewordChange(codeword) } private fun updateIcon() { @@ -241,15 +263,32 @@ class WordInputHandler(wkt9: WKT9Interface, context: WKT9) : InputHandler(wkt9, wkt9.onUpdateStatusIcon(icon) } - private suspend fun queryT9Candidates(codeWord: StringBuilder, limit: Int = 10): List { + private fun undoCodeword() { + undoCandidate() + clearCodeword(false) + } + + private fun undoCandidate() { + val rangeStart = composeRangeStart ?: cursorPosition + val rangeEnd = composeRangeEnd ?: cursorPosition + + // Remove text codeword produced from editor + wkt9.onCompose("", rangeStart, rangeEnd) + + // Remove text codeword produced from content + content.deleteRange(rangeStart, rangeEnd) + + clearCandidates(false) + } + + private suspend fun queryT9Candidates(codeWord: StringBuilder, limit: Int = 10) { val results = wordDao.findCandidates(codeWord.toString(), limit) val capitalize = Capitalize(capMode) - val candidates = mutableListOf() + + candidates.clear() results.forEach { result -> candidates.add(capitalize.word(result.word, sentenceStart)) } - - return candidates } } \ No newline at end of file diff --git a/app/src/main/java/net/mezimmah/wkt9/voice/Whisper.kt b/app/src/main/java/net/mezimmah/wkt9/voice/Whisper.kt index e68ec61..9c57bd6 100644 --- a/app/src/main/java/net/mezimmah/wkt9/voice/Whisper.kt +++ b/app/src/main/java/net/mezimmah/wkt9/voice/Whisper.kt @@ -1,5 +1,18 @@ package net.mezimmah.wkt9.voice +import android.media.MediaRecorder +import android.util.Log +import android.view.View +import android.widget.HorizontalScrollView +import android.widget.LinearLayout +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import net.mezimmah.wkt9.R +import net.mezimmah.wkt9.WKT9 +import net.mezimmah.wkt9.inputmode.InputManager import okhttp3.MediaType.Companion.toMediaType import okhttp3.MultipartBody import okhttp3.OkHttpClient @@ -9,15 +22,113 @@ import java.io.File import java.io.IOException import java.util.concurrent.TimeUnit -class Whisper { +class Whisper( + private val context: WKT9, + private val inputManager: InputManager, + private val ui: View +) { + private val tag = "WKT9" + + private val ioScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private val mainScope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + + private var ioJob: Job? = null + + private var recorder: MediaRecorder? = null + private var recording: File? = null + private val client: OkHttpClient = OkHttpClient.Builder() .connectTimeout(2, TimeUnit.SECONDS) .writeTimeout(5, TimeUnit.SECONDS) .readTimeout(25, TimeUnit.SECONDS) .callTimeout(32, TimeUnit.SECONDS) .build() + fun transcribe() { + stopRecording() - fun run(recording: File): String { + val recording = this.recording ?: return + + showTranscribing() + + ioJob?.cancel() + ioJob = ioScope.launch { + try { + val transcription = run(recording) + + mainScope.launch { + showCandidates() + } + + inputManager.handler?.onInsertText(transcription.plus(" ")) + } catch (e: IOException) { + Log.d(tag, "A failure occurred in the communication with the speech-to-text server", e) + } + } + } + + @Suppress("DEPRECATION") + fun record() { + if (recorder != null) stopRecording() + + showMessage() + + recording = File.createTempFile("recording.3gp", null, context.cacheDir) + recorder = MediaRecorder().also { + it.setAudioSource(MediaRecorder.AudioSource.MIC) + it.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP) + it.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB) + it.setOutputFile(recording) + + try { + it.prepare() + it.start() + } catch (e: Exception) { + Log.d(tag, "Failed to start recording", e) + } + } + } + + private fun showCandidates() { + val candidatesView = ui.findViewById(R.id.suggestion_container) + val loadingView = ui.findViewById(R.id.loading_container) + val messageView = ui.findViewById(R.id.message_container) + + candidatesView.visibility = View.VISIBLE + loadingView.visibility = View.GONE + messageView.visibility = View.GONE + } + + private fun showMessage() { + val candidatesView = ui.findViewById(R.id.suggestion_container) + val loadingView = ui.findViewById(R.id.loading_container) + val messageView = ui.findViewById(R.id.message_container) + + candidatesView.visibility = View.GONE + loadingView.visibility = View.GONE + messageView.visibility = View.VISIBLE + } + + private fun showTranscribing() { + val candidatesView = ui.findViewById(R.id.suggestion_container) + val loadingView = ui.findViewById(R.id.loading_container) + val messageView = ui.findViewById(R.id.message_container) + + candidatesView.visibility = View.GONE + loadingView.visibility = View.VISIBLE + messageView.visibility = View.GONE + } + + private fun stopRecording() { + recorder?.run { + stop() + reset() + release() + } + + recorder = null + } + + private fun run(recording: File): String { val mediaType = "audio/3gpp".toMediaType() val requestBody = MultipartBody.Builder() diff --git a/app/src/main/res/drawable/mic.xml b/app/src/main/res/drawable/mic.xml new file mode 100644 index 0000000..e9da2ce --- /dev/null +++ b/app/src/main/res/drawable/mic.xml @@ -0,0 +1,5 @@ + + + + diff --git a/app/src/main/res/layout/message.xml b/app/src/main/res/layout/message.xml new file mode 100644 index 0000000..cbb4f63 --- /dev/null +++ b/app/src/main/res/layout/message.xml @@ -0,0 +1,29 @@ + + + + + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/suggestions.xml b/app/src/main/res/layout/suggestions.xml index 371d499..443d107 100644 --- a/app/src/main/res/layout/suggestions.xml +++ b/app/src/main/res/layout/suggestions.xml @@ -1,5 +1,5 @@ - + android:orientation="vertical"> + + + + + + + android:orientation="horizontal" + android:visibility="gone"> - \ No newline at end of file + + + + + + + + + + + + + + + \ No newline at end of file