-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmedapp.py
272 lines (225 loc) · 9.51 KB
/
medapp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
import streamlit as st
import streamlit_analytics
import numpy as np
from sentence_transformers import SentenceTransformer
import torch
import json
from typing import List, Dict
import os
from PIL import Image
import faiss
from openai import OpenAI
import pickle
import cv2
from dotenv import load_dotenv
import firebase_admin
from firebase_admin import credentials, firestore
import base64
from mindmap import mindmap_tab_content
# Load environment variables
load_dotenv()
# Set up OpenAI client
key = os.getenv("OPENAI_API_KEY")
if not key:
st.error("OpenAI API key not found. Please set the OPENAI_API_KEY environment variable.")
st.stop()
client = OpenAI(api_key=key)
# Initialize Firebase
firebase_key = os.getenv('FIREBASE_KEY')
if firebase_key:
try:
# Decode the base64 encoded key
decoded_key = base64.b64decode(firebase_key).decode('utf-8')
firebase_key_dict = json.loads(decoded_key)
# Ensure the "type" field is present
if "type" not in firebase_key_dict:
firebase_key_dict["type"] = "service_account"
with open('firebase-key.json', 'w') as f:
json.dump(firebase_key_dict, f)
cred = credentials.Certificate('firebase-key.json')
firebase_admin.initialize_app(cred)
db = firestore.client()
print("Firebase initialized successfully")
except Exception as e:
print(f"Error initializing Firebase: {str(e)}")
else:
print("FIREBASE_KEY environment variable not found")
# Set Streamlit page config
st.set_page_config(page_title="Step Zero", page_icon="⚕️", layout="wide", initial_sidebar_state="expanded")
st.markdown("""
<script src="https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.min.js"></script>
<script>mermaid.initialize({startOnLoad:true});</script>
""", unsafe_allow_html=True)
# Check for GPU availability
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")
# Load SentenceTransformer model
@st.cache_resource
def load_sentence_transformer():
return SentenceTransformer('all-MiniLM-L6-v2').to(device)
model = load_sentence_transformer()
# Load and preprocess video data
@st.cache_data
def load_and_preprocess_data(topic: str):
file_path = os.path.join('data', f'{topic.lower()}_videos.json')
with open(file_path, 'r') as f:
data = json.load(f)
texts = [item['text'] for item in data]
# Check if preprocessed embeddings exist
embeddings_file = f"embeddings_{topic.lower()}.pkl"
print(f"creating {embeddings_file}")
if os.path.exists(embeddings_file):
with open(embeddings_file, 'rb') as f:
embeddings = pickle.load(f)
else:
embeddings = model.encode(texts, convert_to_tensor=True, show_progress_bar=True)
embeddings = embeddings.cpu().numpy()
with open(embeddings_file, 'wb') as f:
pickle.dump(embeddings, f)
# Create FAISS index
index = faiss.IndexFlatL2(embeddings.shape[1])
index.add(embeddings)
return data, index, embeddings
# Retrieve relevant passages using FAISS
def retrieve_passages(query: str, index, embeddings: np.ndarray, video_data: List[Dict], top_k: int = 5) -> List[Dict]:
query_embedding = model.encode([query], convert_to_tensor=True, show_progress_bar=False).cpu().numpy()
D, I = index.search(query_embedding, top_k)
retrieved_passages = []
for idx in I[0]:
passage = video_data[idx]
retrieved_passages.append({
'text': passage['text'],
'video_title': passage['video_title'],
'timestamp': passage['timestamp'],
'video_path': passage['video_path']
})
return retrieved_passages
# Generate answer using OpenAI's GPT-4
def generate_answer(query: str, context: str) -> str:
try:
response = client.chat.completions.create(
model="gpt-4o-mini",
messages=[
{"role": "system", "content": "You are a helpful assistant that answers USMLE Step 1 questions based on the provided context. Always ground your answers in the given context and be concise."},
{"role": "user", "content": f"Context: {context}\n\nQuestion: {query}\n\nAnswer:"}
],
max_tokens=500,
n=1,
stop=None,
temperature=0.2,
)
return response.choices[0].message.content.strip()
except Exception as e:
st.error(f"Error generating answer: {str(e)}")
return "Sorry, I couldn't generate an answer at this time."
# Extract frame from video at specific timestamp
@st.cache_data
def extract_frame(video_path: str, timestamp: float) -> Image.Image:
if not os.path.exists(video_path):
st.error(f"Video file not found: {video_path}")
return None
try:
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
st.error(f"Failed to open video file: {video_path}")
return None
# Get video properties
fps = cap.get(cv2.CAP_PROP_FPS)
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = total_frames / fps if fps > 0 else 0
st.info(f"Video properties: FPS={fps}, Total Frames={total_frames}, Duration={duration:.2f}s")
if timestamp > duration:
st.warning(f"Timestamp {timestamp}s exceeds video duration {duration:.2f}s. Using last frame.")
timestamp = duration
frame_number = int(timestamp * fps)
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_number)
ret, frame = cap.read()
cap.release()
if ret:
return Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
else:
st.warning(f"Could not extract frame at timestamp {timestamp}s (frame {frame_number}) from {video_path}")
return None
except cv2.error as e:
st.error(f"OpenCV error: {str(e)}")
return None
except Exception as e:
st.error(f"An unexpected error occurred: {str(e)}")
return None
# Main Streamlit app
def main():
streamlit_analytics.start_tracking(firestore_key_file="firebase-key.json", firestore_collection_name="counts")
st.title("Step Zero")
st.subheader("Start your journey to Step 1 success with Step Zero!")
# Add new tabs for disclosures and mindmap
tab1, tab2, tab3 = st.tabs(["Main", "Mindmap", "Disclosures"])
topics = ["immunology", "gastroenterology", "cell biology"]
selected_topic = st.sidebar.selectbox("Select a topic", topics, key="topic_selectbox")
video_data, index, embeddings = load_and_preprocess_data(selected_topic)
with tab1:
main_tab_content(video_data, index, embeddings)
with tab2:
mindmap_tab_content()
with tab3:
disclosures_tab_content()
streamlit_analytics.stop_tracking(firestore_key_file="firebase-key.json", firestore_collection_name="counts")
def main_tab_content(video_data, index, embeddings):
# Content for the main tab
user_query = st.text_input("Enter your question:", key="user_query_input")
submit_button = st.button("Submit", key="submit_button")
if submit_button and user_query:
with st.spinner("Searching for relevant information..."):
relevant_passages = retrieve_passages(user_query, index, embeddings, video_data)
context = " ".join([p["text"] for p in relevant_passages])
with st.spinner("Generating answer..."):
answer = generate_answer(user_query, context)
st.subheader("Generated Answer:")
st.write(answer)
# Store values in session state
st.session_state.user_query = user_query
st.session_state.answer = answer
st.session_state.relevant_passages = relevant_passages
with st.expander("View Relevant Passages", expanded=True):
for passage in relevant_passages:
st.write(f"Video: {passage['video_title']}")
st.write(f"Timestamp: {passage['timestamp']}")
st.write(f"Relevant text: {passage['text']}")
frame = extract_frame(passage['video_path'], passage['timestamp'])
if frame:
st.image(frame, caption=f"Frame at {passage['timestamp']} seconds")
else:
st.write("Failed to extract frame from video.")
st.write("---")
# Add a message about the Mindmap
st.info("To create a Mindmap for this query, please go to the Mindmap tab.")
# Add the feedback button at the end of the main tab
st.markdown("---")
st.markdown(
"""
<div style="display: flex; justify-content: center; margin-top: 30px;">
<a href="https://forms.gle/ht6MH14t8kFqrrni6" target="_blank">
<button style="
font-size: 18px;
padding: 12px 24px;
background-color: #FFB347;
color: white;
border: none;
border-radius: 5px;
cursor: pointer;
box-shadow: 0 4px 8px rgba(0,0,0,0.1);
transition: all 0.3s ease;
">
⚕️ Leave Feedback if you liked it! ⚕️
</button>
</a>
</div>
""",
unsafe_allow_html=True
)
def disclosures_tab_content():
st.header("Disclosures")
with open("disclosures.txt", "r") as f:
disclosures_content = f.read()
st.markdown(disclosures_content)
if __name__ == "__main__":
main()