Skip to content

Commit

Permalink
fix: add lang_code support; warn on generative Agents
Browse files Browse the repository at this point in the history
  • Loading branch information
kmaphoenix committed Oct 29, 2024
1 parent 0822934 commit 24d94a9
Showing 1 changed file with 25 additions and 5 deletions.
30 changes: 25 additions & 5 deletions src/dfcx_scrapi/tools/nlu_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def __init__(
creds_path: str = None,
creds_dict: Dict[str, str] = None,
creds=None,
language_code: str = "en"
):

super().__init__(
Expand All @@ -111,15 +112,24 @@ def __init__(
self.agent_id = agent_id
self._sheets_client = self._build_sheets_client()

self._a = agents.Agents(creds=self.creds)
self._i = intents.Intents(creds=self.creds)
self._f = flows.Flows(creds=self.creds)
self._p = pages.Pages(creds=self.creds)
self._a = agents.Agents(creds=self.creds, language_code=language_code)
self._dc = conversation.DialogflowConversation(
creds_path=creds_path, agent_id=agent_id
creds_path=creds_path, agent_id=agent_id,
language_code=language_code
)
self._dffx = dataframe_functions.DataframeFunctions(creds=self.creds)

def get_agent_type(self, agent_id: str):
"""Return the Agent type for logging purposes."""
agent = self._a.get_agent(agent_id)

if agent.start_flow:
return "flow"
elif agent.start_playbook:
return "generative"
else:
raise ValueError("Could not determine Agent type.")

def _build_sheets_client(self):
client = gspread.authorize(self.creds)

Expand Down Expand Up @@ -233,6 +243,16 @@ def run_evals(self, df: pd.DataFrame, chunk_size: int = 300,
"""Run the full Eval dataset."""
logsx = "-" * 10

agent_type = self.get_agent_type(self.agent_id)
warnx = "!" * 10
if agent_type == "generative":
msg = f"{warnx} Agent Type is GENERATIVE {warnx} \nThis specific"\
" class, NluEvals, is optimized for load testing standard"\
" Dialogflow CX Flow based Agents with Intents.\nIf you wish"\
" to evaluate a Generative Agent, please use"\
" dfcx_scrapi.tools.evaluations.Evaluations instead.\n"
logging.warning(msg)

logging.info(f"{logsx} STARTING {eval_run_display_name} {logsx}")
results = self._dc.run_intent_detection(
test_set=df, chunk_size=chunk_size, rate_limit=rate_limit
Expand Down

0 comments on commit 24d94a9

Please sign in to comment.