Skip to content
This repository has been archived by the owner on Jul 7, 2022. It is now read-only.

Perform an action when the user enter a voice input #339

Merged
merged 17 commits into from
May 12, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -4,8 +4,8 @@ import ch.epfl.sdp.mobile.application.speech.SpeechFacade
import ch.epfl.sdp.mobile.application.speech.SpeechFacade.RecognitionResult.*
import ch.epfl.sdp.mobile.test.infrastructure.speech.FailingSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuccessfulSpeechRecognizer
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuccessfulSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuspendingSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.UnknownCommandSpeechRecognizerFactory
import com.google.common.truth.Truth.assertThat
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.launch
@@ -31,7 +31,7 @@ class SpeechFacadeTest {

@Test
fun given_successfulRecognizer_when_recognizes_then_returnsResults() = runTest {
val facade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val facade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
assertThat(facade.recognize()).isEqualTo(Success(SuccessfulSpeechRecognizer.Results))
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package ch.epfl.sdp.mobile.test.infrastructure.speech

import ch.epfl.sdp.mobile.infrastructure.speech.SpeechRecognizer
import ch.epfl.sdp.mobile.infrastructure.speech.SpeechRecognizerFactory

/**
* An implementation of [SpeechRecognizerFactory] which result is recognized but lead to a illegal
* move.
*/
object IllegalActionSpeechRecognizerFactory : SpeechRecognizerFactory {
override fun createSpeechRecognizer() = IllegalActionSpeechRecognizer()
}

class IllegalActionSpeechRecognizer : SpeechRecognizer {

companion object {

/** The results which will always be returned on success. */
val Results = listOf("King a3 to b3", "World")
}

private var listener: SpeechRecognizer.Listener? = null
override fun setListener(listener: SpeechRecognizer.Listener) {
this.listener = listener
}
override fun startListening() {
listener?.onResults(Results)
}
override fun stopListening() = Unit
override fun destroy() = Unit
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package ch.epfl.sdp.mobile.test.infrastructure.speech

import ch.epfl.sdp.mobile.infrastructure.speech.SpeechRecognizer
import ch.epfl.sdp.mobile.infrastructure.speech.SpeechRecognizerFactory

/**
* An implementation of [SpeechRecognizerFactory] which result is recognized but and lead to a legal
* move.
*/
object LegalActionSpeechRecognizerFactory : SpeechRecognizerFactory {
override fun createSpeechRecognizer() = LegalActionSpeechRecognizer()
}

class LegalActionSpeechRecognizer : SpeechRecognizer {

companion object {

/** The results which will always be returned on success. */
val Results = listOf("Pawn e2 to e4", "World")
}

private var listener: SpeechRecognizer.Listener? = null
override fun setListener(listener: SpeechRecognizer.Listener) {
this.listener = listener
}
override fun startListening() {
listener?.onResults(Results)
}
override fun stopListening() = Unit
override fun destroy() = Unit
}
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ import ch.epfl.sdp.mobile.infrastructure.speech.SpeechRecognizer
import ch.epfl.sdp.mobile.infrastructure.speech.SpeechRecognizerFactory

/** An implementation of [SpeechRecognizerFactory] which always succeeds. */
object SuccessfulSpeechRecognizerFactory : SpeechRecognizerFactory {
object UnknownCommandSpeechRecognizerFactory : SpeechRecognizerFactory {
override fun createSpeechRecognizer() = SuccessfulSpeechRecognizer()
}

Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ import ch.epfl.sdp.mobile.state.StatefulSettingsScreen
import ch.epfl.sdp.mobile.test.infrastructure.assets.fake.emptyAssets
import ch.epfl.sdp.mobile.test.infrastructure.persistence.auth.emptyAuth
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.emptyStore
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuccessfulSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.UnknownCommandSpeechRecognizerFactory
import io.mockk.every
import io.mockk.mockk
import kotlinx.coroutines.test.runTest
@@ -41,7 +41,7 @@ class AuthenticatedUserProfileScreenStateTest {
val chessFacade = ChessFacade(auth, store, assets)
val socialFacade = SocialFacade(auth, store)
val authenticationFacade = AuthenticationFacade(auth, store)
val speechFacade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val speechFacade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
val tournamentFacade = TournamentFacade(auth, store)

rule.setContent {
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ import ch.epfl.sdp.mobile.test.infrastructure.assets.fake.emptyAssets
import ch.epfl.sdp.mobile.test.infrastructure.persistence.auth.emptyAuth
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.buildStore
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.document
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuccessfulSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.UnknownCommandSpeechRecognizerFactory
import ch.epfl.sdp.mobile.ui.game.ChessBoardState
import com.google.accompanist.permissions.ExperimentalPermissionsApi
import com.google.common.truth.Truth.assertThat
@@ -38,7 +38,7 @@ class ClassicChessBoardStateTest {
collection("games") { document("id", ChessDocument(whiteId = "id1", blackId = "id2")) }
}
val facade = ChessFacade(auth, store, assets)
val speechFacade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val speechFacade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
val user = mockk<AuthenticatedUser>()
every { user.uid } returns "id1"

Original file line number Diff line number Diff line change
@@ -30,9 +30,7 @@ import ch.epfl.sdp.mobile.test.infrastructure.assets.fake.emptyAssets
import ch.epfl.sdp.mobile.test.infrastructure.persistence.auth.emptyAuth
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.buildStore
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.document
import ch.epfl.sdp.mobile.test.infrastructure.speech.FailingSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuccessfulSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuspendingSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.*
import ch.epfl.sdp.mobile.test.ui.game.ChessBoardRobot
import ch.epfl.sdp.mobile.test.ui.game.click
import ch.epfl.sdp.mobile.test.ui.game.drag
@@ -718,28 +716,49 @@ class StatefulGameScreenTest {
}

@Test
fun given_successfulRecognizer_when_clicksListening_then_displaysRecognitionResults() {
// This will fail once we want to move the pieces instead.
fun given_successfulRecognizer_when_clicksListening_then_displaysUnknownCmdResults() {
val robot =
emptyGameAgainstOneselfRobot(
recognizer = SuccessfulSpeechRecognizerFactory,
recognizer = UnknownCommandSpeechRecognizerFactory,
audioPermission = GrantedPermissionState,
)
robot.onNodeWithLocalizedContentDescription { gameMicOffContentDescription }.performClick()
// Print null because the input in not recognized
robot.onNodeWithText("null").assertExists()
robot.onNodeWithText(robot.strings.gameSnackBarUnknownCommand).assertExists()
}

@Test
fun given_failingRecognizer_when_clicksListening_then_displaysFailedRecognitionResults() {
// This will fail once we want to move the pieces instead.
val robot =
emptyGameAgainstOneselfRobot(
recognizer = FailingSpeechRecognizerFactory,
audioPermission = GrantedPermissionState,
)
robot.onNodeWithLocalizedContentDescription { gameMicOffContentDescription }.performClick()
robot.onNodeWithText("Internal failure").assertExists()
robot.onNodeWithText(robot.strings.gameSnackBarInternalFailure).assertExists()
}

@Test
fun given_legalActionRecognizer_when_clicksListening_then_noneMessagesDisplayed() {
val robot =
emptyGameAgainstOneselfRobot(
recognizer = LegalActionSpeechRecognizerFactory,
audioPermission = GrantedPermissionState,
)
robot.onNodeWithLocalizedContentDescription { gameMicOffContentDescription }.performClick()
robot.onNodeWithText(robot.strings.gameSnackBarIllegalAction).assertDoesNotExist()
robot.onNodeWithText(robot.strings.gameSnackBarUnknownCommand).assertDoesNotExist()
robot.onNodeWithText(robot.strings.gameSnackBarInternalFailure).assertDoesNotExist()
}

@Test
fun given_illegalActionRecognizer_when_clicksListening_then_displayIllegalActionErrorMsg() {
val robot =
emptyGameAgainstOneselfRobot(
recognizer = IllegalActionSpeechRecognizerFactory,
audioPermission = GrantedPermissionState,
)
robot.onNodeWithLocalizedContentDescription { gameMicOffContentDescription }.performClick()
robot.onNodeWithText(robot.strings.gameSnackBarIllegalAction).assertExists()
}

@Test
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.buildStore
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.document
import ch.epfl.sdp.mobile.test.infrastructure.persistence.store.emptyStore
import ch.epfl.sdp.mobile.test.infrastructure.speech.FailingSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.SuccessfulSpeechRecognizerFactory
import ch.epfl.sdp.mobile.test.infrastructure.speech.UnknownCommandSpeechRecognizerFactory
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.test.runTest
@@ -449,7 +449,7 @@ class StatefulHomeTest {
val authFacade = AuthenticationFacade(auth, store)
val chessFacade = ChessFacade(auth, store, assets)
val socialFacade = SocialFacade(auth, store)
val speechFacade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val speechFacade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
val tournamentFacade = TournamentFacade(auth, store)

authFacade.signInWithEmail("[email protected]", "password")
@@ -492,7 +492,7 @@ class StatefulHomeTest {
val authFacade = AuthenticationFacade(auth, store)
val chessFacade = ChessFacade(auth, store, assets)
val socialFacade = SocialFacade(auth, store)
val speechFacade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val speechFacade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
val tournamentFacade = TournamentFacade(auth, store)

authFacade.signInWithEmail("[email protected]", "password")
@@ -632,7 +632,7 @@ class StatefulHomeTest {
val authFacade = AuthenticationFacade(auth, store)
val chessFacade = ChessFacade(auth, store, assets)
val socialFacade = SocialFacade(auth, store)
val speechFacade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val speechFacade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
val tournamentFacade = TournamentFacade(auth, store)

authFacade.signUpWithEmail("[email protected]", "user", "password")
@@ -764,7 +764,7 @@ class StatefulHomeTest {
val authFacade = AuthenticationFacade(auth, store)
val chessFacade = ChessFacade(auth, store, assets)
val socialFacade = SocialFacade(auth, store)
val speechFacade = SpeechFacade(SuccessfulSpeechRecognizerFactory)
val speechFacade = SpeechFacade(UnknownCommandSpeechRecognizerFactory)
val tournamentFacade = TournamentFacade(auth, store)

val user = authFacade.currentUser.filterIsInstance<AuthenticatedUser>().first()
Original file line number Diff line number Diff line change
@@ -6,12 +6,11 @@ import androidx.compose.ui.test.onNodeWithContentDescription
import androidx.compose.ui.test.onNodeWithText
import androidx.compose.ui.test.performClick
import ch.epfl.sdp.mobile.test.state.setContentWithLocalizedStrings
import ch.epfl.sdp.mobile.ui.game.*
import ch.epfl.sdp.mobile.ui.game.ChessBoardState.Piece
import ch.epfl.sdp.mobile.ui.game.GameScreen
import ch.epfl.sdp.mobile.ui.game.GameScreenState
import ch.epfl.sdp.mobile.ui.game.MovableChessBoardState
import ch.epfl.sdp.mobile.ui.game.MovesInfoState.Move
import ch.epfl.sdp.mobile.ui.game.PlayersInfoState
import ch.epfl.sdp.mobile.ui.game.SpeechRecognizerState.SpeechRecognizerError
import ch.epfl.sdp.mobile.ui.game.SpeechRecognizerState.SpeechRecognizerError.*
import org.junit.Rule
import org.junit.Test

@@ -21,6 +20,9 @@ class GameScreenTest {
private class SnapshotGameScreenState :
GameScreenState<Piece>,
MovableChessBoardState<Piece> by ChessBoardTest.SinglePieceSnapshotChessBoardState() {

override var currentError: SpeechRecognizerError = None

override val moves: List<Move>
get() =
listOf(
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ import ch.epfl.sdp.mobile.application.authentication.AuthenticatedUser
import ch.epfl.sdp.mobile.application.chess.Match
import ch.epfl.sdp.mobile.state.game.ActualGameScreenState
import ch.epfl.sdp.mobile.ui.game.*
import ch.epfl.sdp.mobile.ui.game.SpeechRecognizerState.SpeechRecognizerError
import com.google.accompanist.permissions.ExperimentalPermissionsApi
import com.google.accompanist.permissions.PermissionState
import com.google.accompanist.permissions.rememberPermissionState
@@ -67,6 +68,8 @@ fun StatefulGameScreen(

StatefulPromoteDialog(gameScreenState)

StatefulSnackBar(gameScreenState, snackbarHostState)

GameScreen(
state = gameScreenState,
modifier = modifier,
@@ -99,3 +102,30 @@ fun StatefulPromoteDialog(
)
}
}

/**
* A composable which displays a snackbar to show the [SpeechRecognizerState.currentError] when it's
* modified
*
* @param state the [SpeechRecognizerState] which backs this dialog.
* @param snackbarHostState the [SnackbarHostState] used to display some results.
*/
@Composable
private fun StatefulSnackBar(
state: SpeechRecognizerState,
snackbarHostState: SnackbarHostState,
) {
matt989253 marked this conversation as resolved.
Show resolved Hide resolved

val strings = LocalLocalizedStrings.current
LaunchedEffect(state.currentError) {
val msg =
when (state.currentError) {
SpeechRecognizerError.InternalError -> strings.gameSnackBarInternalFailure
SpeechRecognizerError.IllegalAction -> strings.gameSnackBarIllegalAction
SpeechRecognizerError.UnknownCommand -> strings.gameSnackBarUnknownCommand
SpeechRecognizerError.None -> return@LaunchedEffect // Nothing to show
}
snackbarHostState.showSnackbar(msg)
state.currentError = SpeechRecognizerError.None
}
}
Original file line number Diff line number Diff line change
@@ -8,11 +8,11 @@ import androidx.compose.material.SnackbarHostState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
import ch.epfl.sdp.mobile.application.chess.engine.Position
import ch.epfl.sdp.mobile.application.chess.voice.VoiceInput
import ch.epfl.sdp.mobile.application.speech.SpeechFacade
import ch.epfl.sdp.mobile.state.game.core.MutableGameDelegate
import ch.epfl.sdp.mobile.ui.game.SpeechRecognizerState
import ch.epfl.sdp.mobile.ui.game.SpeechRecognizerState.*
import com.google.accompanist.permissions.ExperimentalPermissionsApi
import com.google.accompanist.permissions.PermissionState
import kotlinx.coroutines.CoroutineScope
@@ -37,6 +37,8 @@ constructor(
private val scope: CoroutineScope,
) : SpeechRecognizerState {

override var currentError by mutableStateOf(SpeechRecognizerError.None)

override var listening: Boolean by mutableStateOf(false)
private set

@@ -59,17 +61,20 @@ constructor(
when (val speech = facade.recognize()) {
// TODO : Display an appropriate message, otherwise act on the board.
SpeechFacade.RecognitionResult.Failure.Internal ->
snackbarHostState.showSnackbar("Internal failure")
currentError = SpeechRecognizerError.InternalError
is SpeechFacade.RecognitionResult.Success -> {

val parsedValue = VoiceInput.parseInput(speech.results)
snackbarHostState.showSnackbar(parsedValue.toString())
// Parsed the input
val parsedAction = VoiceInput.parseInput(speech.results)
if (parsedAction != null) {
// Try to perform the action
val isSuccessful = delegate.tryPerformAction(parsedAction)

// TODO(Chau) : Do something more interesting
Position.all()
.flatMap { delegate.game.actions(it) }
.onEach { delegate.tryPerformAction(it) }
.firstOrNull()
// If the action is illegal
if (!isSuccessful) currentError = SpeechRecognizerError.IllegalAction
} else { // If cannot be parsed
currentError = SpeechRecognizerError.UnknownCommand
}
BadrTad marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
Original file line number Diff line number Diff line change
@@ -6,6 +6,28 @@ import androidx.compose.runtime.Stable
@Stable
interface SpeechRecognizerState {

/**
* Store the current error value, if there are not error involved the value is set to
* [SpeechRecognizerError.None]
*/
var currentError: SpeechRecognizerError

/** Enum class that defined different error related to the speech recognizer */
enum class SpeechRecognizerError {

// FIXME General error, temporary
InternalError,

/** The asked command cannot be performed */
IllegalAction,

/** The command cannot be parsed */
UnknownCommand,

/** None error */
None,
}

/** A [Boolean] which indicates if the device is currently listening to voice inputs. */
val listening: Boolean

Loading