Skip to content

Commit

Permalink
fix: remove unused code; add Sessions updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@kmaphoenix
kmaphoenix committed Oct 18, 2023
1 parent 2e177ee commit a277f55
Showing 1 changed file with 109 additions and 110 deletions.
219 changes: 109 additions & 110 deletions examples/vertex_ai_conversation/evaluation_tool__numeric_score__colab.ipynb
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,10 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "WpkyirmC-F33"
},
"source": [
"# Vertex AI Conversation - Evaluation Tool\n",
"\n",
Expand Down Expand Up @@ -50,19 +39,16 @@
"* **0** : Neutral answer > no answer or answer contains irrelevant info\n",
"* **-1** : Hurtful answer > wrong or misleading info, or inappropriate tone\n",
"\n"
],
"metadata": {
"id": "WpkyirmC-F33"
}
]
},
{
"cell_type": "markdown",
"source": [
"## Set-up\n"
],
"metadata": {
"id": "Afvsuux0zaWZ"
}
},
"source": [
"## Set-up\n"
]
},
{
"cell_type": "code",
Expand All @@ -83,81 +69,76 @@
"from dfcx_scrapi.core.sessions import Sessions\n",
"from google.auth import default\n",
"from google.colab import auth\n",
"from google.colab import files\n",
"from google.protobuf.json_format import MessageToDict\n",
"\n",
"\n",
"def get_agent_answer(user_query):\n",
" s = Sessions()\n",
" session_id = s.build_session_id(agent_id)\n",
" res = MessageToDict(s.detect_intent(agent_id, session_id, user_query)._pb)\n",
"\n",
" answer_text = res['responseMessages'][0]['text']['text'][0]\n",
" answer_link = res['responseMessages'][1]['payload']['richContent'][0][0]['actionLink'] if len(res['responseMessages']) > 1 else ''\n",
"\n",
" return f\"{answer_text} ({answer_link})\"\n"
"from google.colab import files"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "sztyBjNlIGAw"
},
"outputs": [],
"source": [
"# Authentication\n",
"\n",
"auth.authenticate_user()\n",
"creds, _ = default()\n"
],
"metadata": {
"id": "sztyBjNlIGAw"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "mRUB0Uf-3uzS"
},
"outputs": [],
"source": [
"# Agent config\n",
"project_id = '' #@param{type: 'string'}\n",
"location = 'global' #@param{type: 'string'}\n",
"agent_id = '' #@param{type: 'string'}\n",
"\n",
"agent_id = f\"projects/{project_id}/locations/{location}/agents/{agent_id}\"\n",
"print(agent_id)\n"
],
"metadata": {
"id": "mRUB0Uf-3uzS"
},
"execution_count": null,
"outputs": []
"print(agent_id)\n",
"\n",
"s = Sessions(agent_id=agent_id)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "OChJbblt3dt7"
},
"outputs": [],
"source": [
"# Test\n",
"user_query = 'Hello World!'\n",
"agent_answer = get_agent_answer(user_query)\n",
"agent_answer = s.get_agent_answer(user_query)\n",
"print(f\" Q: {user_query}\\n A: {agent_answer}\")\n",
"\n",
"user_query = 'Which is the cheapest plan?'\n",
"agent_answer = get_agent_answer(user_query)\n",
"agent_answer = s.get_agent_answer(user_query)\n",
"print(f\" Q: {user_query}\\n A: {agent_answer}\")\n"
],
"metadata": {
"id": "OChJbblt3dt7"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"## Generate Questions & Answer"
],
"metadata": {
"id": "L2WQime-8-Dw"
}
},
"source": [
"## Generate Questions & Answer"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "q3II66B04F0j"
},
"outputs": [],
"source": [
"# Create sample csv\n",
"\n",
Expand All @@ -177,15 +158,15 @@
"file_name = 'data_sample.csv'\n",
"sample_df.to_csv(file_name, encoding='utf-8-sig', index=False)\n",
"files.download(file_name)\n"
],
"metadata": {
"id": "q3II66B04F0j"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "OYr4Dy77KbfL"
},
"outputs": [],
"source": [
"input(f\"In your local drive, you can find the csv file '{file_name}' Add the user_query and ideal_answer per example \\nWhen done, click 'Enter'\")\n",
"print('done')\n",
Expand All @@ -199,52 +180,52 @@
"assert set(df.columns) == set(sample_df.columns), f\"The csv must have the following columns: {sample_df.columns.values}\"\n",
"\n",
"df"
],
"metadata": {
"id": "OYr4Dy77KbfL"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "RmJcxpFI881j"
},
"outputs": [],
"source": [
"# Generate answers for each query\n",
"df['agent_answer'] = df.apply(lambda row: get_agent_answer(row[\"user_query\"]), axis=1)\n",
"df['agent_answer'] = df.apply(lambda row: s.get_agent_answer(row[\"user_query\"]), axis=1)\n",
"\n",
"df"
],
"metadata": {
"id": "RmJcxpFI881j"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"# Rating"
],
"metadata": {
"id": "yO2x7lc2BRDR"
}
},
"source": [
"# Rating"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "ZfAMlQbS8qsy"
},
"outputs": [],
"source": [
"# Export to local drive as csv file\n",
"file_name = 'output.csv'\n",
"df.to_csv(file_name, encoding='utf-8-sig', index=False)\n",
"files.download(file_name)\n"
],
"metadata": {
"id": "ZfAMlQbS8qsy"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "SEU44Mcy9mBU"
},
"outputs": [],
"source": [
"input(f\"In your local drive, you can find the csv file '{file_name}' Rate each agent_answer using ideal_answer as reference. Rating from -1 to 3. \\nWhen done, click 'Enter'\")\n",
"print('done')\n",
Expand All @@ -258,24 +239,24 @@
"assert set(df.columns) == set(sample_df.columns), f\"The csv must have the following columns: {sample_df.columns.values}\"\n",
"\n",
"df"
],
"metadata": {
"id": "SEU44Mcy9mBU"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"# Results\n"
],
"metadata": {
"id": "W5j9yAewRmNO"
}
},
"source": [
"# Results\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "I5209MB7VS1q"
},
"outputs": [],
"source": [
"# Rating distribution\n",
"#df[\"rating\"].describe()\n",
Expand Down Expand Up @@ -314,21 +295,39 @@
"\n",
"fig.savefig('ratings_distribution.png', dpi=fig.dpi)\n",
"\n"
],
"metadata": {
"id": "I5209MB7VS1q"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"source": [],
"execution_count": null,
"metadata": {
"id": "rYwsIZ0Ej-v9"
},
"execution_count": null,
"outputs": []
"outputs": [],
"source": []
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
]
}
},
"nbformat": 4,
"nbformat_minor": 0
}

0 comments on commit a277f55

Please sign in to comment.