Skip to content

Commit

Permalink
Feat: add live interaction
Browse files Browse the repository at this point in the history
  • Loading branch information
@jurmy24
jurmy24 committed Sep 21, 2024
1 parent a4cbd2d commit c80ab77
Show file tree
Hide file tree
Showing 3 changed files with 149 additions and 75 deletions.
4 changes: 4 additions & 0 deletions Utter.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
E87D53062C9F567C0067A1A0 /* LiveInteractionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E87D53052C9F567C0067A1A0 /* LiveInteractionView.swift */; };
E87D53082C9F59D80067A1A0 /* LiveInteractionViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = E87D53072C9F59D80067A1A0 /* LiveInteractionViewModel.swift */; };
E87D530B2C9F5A650067A1A0 /* Vapi in Frameworks */ = {isa = PBXBuildFile; productRef = E87D530A2C9F5A650067A1A0 /* Vapi */; };
E87D530D2C9F5D8A0067A1A0 /* TokensInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = E87D530C2C9F5D8A0067A1A0 /* TokensInfo.plist */; };
E8B5A72C2C878BA700B27287 /* FirebaseAnalytics in Frameworks */ = {isa = PBXBuildFile; productRef = E8B5A72B2C878BA700B27287 /* FirebaseAnalytics */; };
E8E346E82C9816A6006DDA92 /* AvatarGrid.swift in Sources */ = {isa = PBXBuildFile; fileRef = E8E346E72C9816A6006DDA92 /* AvatarGrid.swift */; };
E8E346EB2C982DB2006DDA92 /* LanguageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E8E346EA2C982DB2006DDA92 /* LanguageView.swift */; };
Expand Down Expand Up @@ -148,6 +149,7 @@
E87D53012C9F247F0067A1A0 /* PronunciationViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PronunciationViewModel.swift; sourceTree = "<group>"; };
E87D53052C9F567C0067A1A0 /* LiveInteractionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LiveInteractionView.swift; sourceTree = "<group>"; };
E87D53072C9F59D80067A1A0 /* LiveInteractionViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LiveInteractionViewModel.swift; sourceTree = "<group>"; };
E87D530C2C9F5D8A0067A1A0 /* TokensInfo.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = TokensInfo.plist; sourceTree = "<group>"; };
E8E346E72C9816A6006DDA92 /* AvatarGrid.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AvatarGrid.swift; sourceTree = "<group>"; };
E8E346EA2C982DB2006DDA92 /* LanguageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LanguageView.swift; sourceTree = "<group>"; };
E8E346EC2C982F7B006DDA92 /* LanguageOption.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LanguageOption.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -257,6 +259,7 @@
children = (
E82E3E4D2C885BFF00EF9A6A /* GoogleService-Info.plist */,
E83960A42C6BF90400D8456D /* Info.plist */,
E87D530C2C9F5D8A0067A1A0 /* TokensInfo.plist */,
E83960952C6BEA1400D8456D /* UtterApp.swift */,
E827E4A42C90B2A600D6D4F1 /* Launch Screen.storyboard */,
E87D52F72C9DD3130067A1A0 /* Managers */,
Expand Down Expand Up @@ -667,6 +670,7 @@
E839609D2C6BEA1500D8456D /* Preview Assets.xcassets in Resources */,
E827E4A52C90B2A600D6D4F1 /* Launch Screen.storyboard in Resources */,
E839609A2C6BEA1500D8456D /* Assets.xcassets in Resources */,
E87D530D2C9F5D8A0067A1A0 /* TokensInfo.plist in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
202 changes: 130 additions & 72 deletions Utter/Core/Story/Components/Exercise/ExerciseTypes/LiveInteractionView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,51 @@ class CallManager: ObservableObject {
enum CallState {
case started, loading, ended
}

@Published var callState: CallState = .ended
var vapiEvents = [Vapi.Event]()
private var cancellables = Set<AnyCancellable>()
let vapi: Vapi

@Published var progress: Double = 0
private var timer: Timer?

func startTimer() {
progress = 0
timer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in
guard let self = self else { return }
if self.progress < 1.0 {
self.progress += 1.0 / 60.0 // Increase by 1/60 each second
} else {
self.stopTimer()
}
}
}

func stopTimer() {
timer?.invalidate()
timer = nil
progress = 0
}


// TODO: This should be run on my own server as its really bad practice to put in the app bundle
init() {
vapi = Vapi(
publicKey: "<Your Vapi Public Key>"
)
// Load the plist file
if let path = Bundle.main.path(forResource: "TokensInfo", ofType: "plist"),
let plistData = NSDictionary(contentsOfFile: path),
let publicKey = plistData["VAPI_PUBLIC_KEY"] as? String {

// Initialize vapi with the fetched public key
vapi = Vapi(
publicKey: publicKey
)
} else {
// Handle the case where the plist could not be loaded or the key is missing
fatalError("Unable to load public key from TokensInfo.plist")
}
}

func setupVapi() {
vapi.eventPublisher
.sink { [weak self] event in
Expand Down Expand Up @@ -52,96 +85,118 @@ class CallManager: ObservableObject {
}
.store(in: &cancellables)
}

@MainActor
func handleCallAction() async {
func handleCallAction(firstMessage: String, extraContext: String) async {
if callState == .ended {
await startCall()
await startCall(firstMessage: firstMessage, extraContext: extraContext)
} else {
endCall()
}
}

@MainActor
func startCall() async {
func startCall(firstMessage: String, extraContext: String) async {
callState = .loading
let assistant = [
"model": [
"provider": "openai",
"model": "gpt-3.5-turbo",
"messages": [
["role":"system", "content":"You are an assistant."]
],
],
"firstMessage": "Hey there",
"voice": "jennifer-playht"

let context = "STORY HISTORY:\n" + extraContext + "\nGUIDANCE: Make sure the user speaks the language of the story and help them as needed. Speak in simple terminology in the language. You can also provide hints if they are struggling."

let overrides = [
"firstMessage": firstMessage,
"context": context,
] as [String : Any]
do {
try await vapi.start(assistant: assistant)
try await vapi.start(assistantId: "77c55889-6355-45e7-ac59-bd040ca3a14e", assistantOverrides: overrides)
startTimer() // Start the timer when the call starts
} catch {
print("Error starting call: \(error)")
callState = .ended
}
}

func endCall() {
stopTimer() // Stop the timer when the call ends
vapi.stop()
}
}

struct LiveInteractionView: View {
@Binding var isInteractive: Bool
@StateObject private var callManager = CallManager()

@State private var isAnimating = false
let firstMessage: String
let extraContext: String?

var body: some View {
ZStack {
// Background gradient
LinearGradient(gradient: Gradient(colors: [Color("AppBackground").opacity(1), Color("AccentColor").opacity(1)]), startPoint: .top, endPoint: .bottom)
.edgesIgnoringSafeArea(.all)

VStack(spacing: 20) {

Spacer()

Text(callManager.callStateText)
.font(.title)
.fontWeight(.semibold)
.foregroundColor(.white)
.padding()
.background(callManager.callStateColor)
.cornerRadius(10)

Spacer()
VStack {
ZStack {
// Background gradient
LinearGradient(gradient: Gradient(colors: [Color("AppBackgroundColor").opacity(1), Color("AccentColor").opacity(1)]), startPoint: .top, endPoint: .bottom)
.edgesIgnoringSafeArea(.all)

VStack(spacing: 20) {

Spacer()

if callManager.callState == .started {
Circle()
.fill(Color.green.opacity(0.6))
.frame(width: 100, height: 100)
.scaleEffect(isAnimating ? 1.2 : 1.0)
.opacity(isAnimating ? 0.6 : 1.0)
.animation(Animation.easeInOut(duration: 0.8).repeatForever(autoreverses: true), value: isAnimating)
.onAppear {
isAnimating = true
}
.onDisappear {
isAnimating = false
}
} else {
Text(callManager.callStateText)
.font(.title)
.fontWeight(.semibold)
.foregroundColor(.white)
.padding()
.background(callManager.callStateColor)
.cornerRadius(10)
}

Button(action: {
Task {
await callManager.handleCallAction()
Spacer()

Button(action: {
Task {
await callManager.handleCallAction(firstMessage: firstMessage, extraContext: extraContext ?? "")
}
}) {
Text(callManager.buttonText)
.font(.title2)
.fontWeight(.bold)
.foregroundColor(.white)
.padding()
.frame(maxWidth: .infinity)
.background(callManager.buttonColor)
.cornerRadius(10)
}
}) {
Text(callManager.buttonText)
.font(.title2)
.fontWeight(.bold)
.foregroundColor(.white)
.padding()
.frame(maxWidth: .infinity)
.background(callManager.buttonColor)
.cornerRadius(10)
.disabled(callManager.callState == .loading)
.padding(.horizontal, 20)
}
.disabled(callManager.callState == .loading)
.padding(.horizontal, 20)
}
}
.toolbar {
ToolbarItem(placement: .navigationBarLeading) {

}
ToolbarItem(placement: .navigationBarTrailing) {
Button {
callManager.endCall()
isInteractive = false
} label: {
Image(systemName: "xmark")
.font(.headline)
HStack {
if callManager.callState == .started {
ProgressView(value: callManager.progress, total: 1.0)
.progressViewStyle(LinearProgressViewStyle())
.frame(width: 100)
}
Button {
callManager.endCall()
isInteractive = false
} label: {
Image(systemName: "xmark")
.font(.headline)
}
}
}
}
Expand All @@ -154,29 +209,32 @@ struct LiveInteractionView: View {
extension CallManager {
var callStateText: String {
switch callState {
case .started: return "Call in Progress"
case .started: return "..Active.."
case .loading: return "Connecting..."
case .ended: return "Call Off"
case .ended: return "Press start to connect"
}
}

var callStateColor: Color {
switch callState {
case .started: return .green.opacity(0.8)
case .loading: return .orange.opacity(0.8)
case .ended: return .gray.opacity(0.8)
}
}

var buttonText: String {
callState == .loading ? "Loading..." : (callState == .ended ? "Start Call" : "End Call")
callState == .loading ? "Loading..." : (callState == .ended ? "Start" : "Stop")
}

var buttonColor: Color {
callState == .loading ? .gray : (callState == .ended ? .green : .red)
callState == .loading ? .gray : (callState == .ended ? Color("LogoBackgroundColor") : .red)
}
}

#Preview {
LiveInteractionView(isInteractive: .constant(true))
NavigationStack{
LiveInteractionView(isInteractive: .constant(false), firstMessage: "Potato", extraContext: "Nuggets")
}

}
18 changes: 15 additions & 3 deletions Utter/Core/Story/Components/Exercise/ExerciseTypes/SpeakingView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ struct SpeakingView: View {
@StateObject private var viewModel: LineViewModel
@State private var transcribedText: String = ""
@State private var isInteractive: Bool = false
private var processedStory: String = ""

init(exercise: ExerciseOption,
story: StoryData? = nil,
Expand All @@ -31,6 +32,7 @@ struct SpeakingView: View {
self._isExerciseCompleted = isExerciseCompleted
self._isExpandedAfterCompletion = isExpandedAfterCompletion
self._viewModel = StateObject(wrappedValue: LineViewModel(story: story, affectedLine: exercise.affectedLine))
self.processedStory = "" // TODO: create a function to retrieve the story we have processed so far
}

var body: some View {
Expand Down Expand Up @@ -67,8 +69,18 @@ struct SpeakingView: View {
.foregroundColor(speechRecognitionManager.isRecording ? Color.red : (isExerciseCompleted ? Color.gray : colors.accent))
.cornerRadius(8)
}
.fullScreenCover(isPresented: $isInteractive, onDismiss: endSession) {
LiveInteractionView(isInteractive: $isInteractive)
.fullScreenCover(isPresented: $isInteractive , onDismiss: endSession) {
if let query = exercise.query {
NavigationStack{
LiveInteractionView(isInteractive: $isInteractive, firstMessage: query, extraContext: self.processedStory)
}
} else {
LiveInteractionView(isInteractive: $isInteractive, firstMessage: "", extraContext: "")
.onAppear {
print("Issue")
isInteractive = false
}
}
}
}

Expand Down Expand Up @@ -99,7 +111,7 @@ struct SpeakingView: View {
}

private func handleLiveInteraction() {
isInteractive = false
isInteractive = true
}

// Helper function to get the label text and image based on exercise type
Expand Down

0 comments on commit c80ab77

Please sign in to comment.