From 0f0bdbcd8b781b681b710a61b18e2944cf5a7f87 Mon Sep 17 00:00:00 2001 From: jasperan <23caj23@gmail.com> Date: Sun, 10 Mar 2024 22:19:09 +0100 Subject: [PATCH] feat: added summarize from file functionality --- output.txt | 48 +++++++++++++++--------------- summarize_data.txt | 53 +++++++++++++++++++++++++++++++++ summarize_from_file.py | 67 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 24 deletions(-) create mode 100644 summarize_data.txt create mode 100644 summarize_from_file.py diff --git a/output.txt b/output.txt index 074f4ad..4a0c3d8 100644 --- a/output.txt +++ b/output.txt @@ -1,25 +1,25 @@ -/leptonai/search_with_lepton -/rasbt/LLMs-from-scratch -/DataTalksClub/data-engineering-zoomcamp -/farcasterxyz/fc-polls -/bclavie/RAGatouille -/AUTOMATIC1111/stable-diffusion-webui -/nicknochnack/MLTradingBot -/cheahjs/palworld-save-tools -/FuelLabs/fuel-core -/KaisenAmin/c_std -/kroma-network/tachyon +/maybe-finance/maybe +/FujiwaraChoki/MoneyPrinter +/zanfranceschi/rinha-de-backend-2024-q1 +/mitsuhiko/rye /lizongying/my-tv -/100xdevs-cohort-2/assignments -/zed-industries/zed -/danielmiessler/SecLists -/termux/termux-app -/vizia/vizia -/EternalWraith/PalEdit -/trimstray/the-book-of-secret-knowledge -/swordbluesword/PalWorld-NetCrack -/practical-tutorials/project-based-learning -/TaskingAI/TaskingAI -/FuelLabs/fuels-ts -/nvim-lua/kickstart.nvim -/zedeus/nitter \ No newline at end of file +/maybe-finance/maybe-archive +/Awesome-Windows/Awesome +/logseq/logseq +/apple/pkl +/thu-vu92/local-llms-analyse-finance +/apernet/OpenGFW +/tauri-apps/tauri +/netbox-community/netbox +/atopile/atopile +/GrowingGit/GitHub-Chinese-Top-Charts +/FuelLabs/sway +/mui/mui-x +/ansible/ansible +/backstage/backstage +/X-PLUG/MobileAgent +/reqable/re-editor +/home-assistant/core +/grpc/grpc +/divyam234/teldrive +/donnemartin/system-design-primer \ No newline at end of file diff --git a/summarize_data.txt b/summarize_data.txt new file mode 100644 index 0000000..8ce4cea --- /dev/null +++ b/summarize_data.txt @@ -0,0 +1,53 @@ +Language allows you to perform sophisticated text analysis at scale. Using the pretrained and custom models, you can process unstructured text to extract insights without data science expertise. Pretrained models include sentiment analysis, key phrase extraction, text classification, and named entity recognition. You can also train custom models for named entity recognition and text classification with domain specific datasets. Additionally, you can translate text across numerous languages. + +Language Overview + +Language is server-less and multi-tenant service that's accessible using REST API calls. With pretrained and custom models, you can process unstructured text and extract insights without data science expertise. + +The Language pretrained models are frequently retrained and monitored to provide you with the best results. + +You can automate sophisticated text analysis at scale without any machine learning expertise. + +The Language service contains these pretrained language processing capabilities: + +Language Detection + Detects languages based on the provided text, and includes a confidence score. +Text Classification + + Identifies the document category and subcategory that the text belongs to. +Named Entity Recognition + + Identifies common entities, people, places, locations, email, and so on. +Key Phrase Extraction + + Extracts an important set of phrases from a block of text. +Sentiment Analysis + + Identifies aspects from the provided text and classifies each into positive, negative, or neutral polarity. +Text Translation + + Translates text into the language of your choice. +Personal Identifiable Information + + Personal Identifiable Information (PII) detection identifies, classifies, and de-identifies private information in unstructured text + +Creating custom models can be done using Named Entity Recognition and Text Classification. +Ways to Access Language + +You access Language using the Console, REST API, SDKs, or CLI. + +Use any of the following options, based on your preference and its suitability for the task you want to complete: + + The OCI Console is an easy-to-use, browser-based interface. To access the Console, you must use a supported browser. + The REST APIs provide the most functionality, but require programming expertise. API reference and endpoints provide endpoint details and links to the available API reference documents including the Artificial Intelligence Services REST API. + OCI provides SDKs that interact with Language without the need to create a framework. + The CLI provides both quick access and full functionality without the need for programming. + +Regions and Availability Domains + +OCI services are hosted in regions and availability domains. A region +is a localized geographic area, and an availability domain + +is one or more data centers located in that region. + +Language is hosted in OCI commercial and government regions. \ No newline at end of file diff --git a/summarize_from_file.py b/summarize_from_file.py new file mode 100644 index 0000000..208c4e2 --- /dev/null +++ b/summarize_from_file.py @@ -0,0 +1,67 @@ +# coding: utf-8 +# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. +# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license. + +########################################################################## +# summarize_text_demo.py +# Supports Python 3 +########################################################################## +# Info: +# Get texts from LLM model for given prompts using OCI Generative AI Service. +########################################################################## +# Application Command line(no parameter needed) +# python summarize_text_demo.py +########################################################################## + +import oci +import yaml +#import logging +#logging.getLogger('oci').setLevel(logging.DEBUG) + +def main(summary_txt: str = "") -> None: + + with open('config.yaml', 'r') as file: + config_data = yaml.safe_load(file) + + compartment_id = config_data['compartment_id'] + CONFIG_PROFILE = config_data['config_profile'] + config = oci.config.from_file('~/.oci/config', CONFIG_PROFILE) + + # Service endpoint + endpoint = "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" + + generative_ai_inference_client = oci.generative_ai_inference.GenerativeAiInferenceClient(config=config, service_endpoint=endpoint, retry_strategy=oci.retry.NoneRetryStrategy(), timeout=(10,240)) + + # You can also load the summary text from a file, or as a parameter in main + #with open('files/summarize_data.txt', 'r') as file: + # text_to_summarize = file.read() + + summarize_text_detail = oci.generative_ai_inference.models.SummarizeTextDetails() + summarize_text_detail.serving_mode = oci.generative_ai_inference.models.OnDemandServingMode(model_id="cohere.command") + summarize_text_detail.compartment_id = compartment_id + #summarize_text_detail.input = text_to_summarize + + with open('summarize_data.txt', 'r') as file: + text_to_summarize = file.read() + summarize_text_detail.input = text_to_summarize + + summarize_text_detail.additional_command = "Generate a teaser summary for this Markdown file. Share an interesting insight to captivate attention." + summarize_text_detail.extractiveness = "AUTO" # HIGH, LOW + summarize_text_detail.format = "AUTO" # brackets, paragraph + summarize_text_detail.length = "LONG" # high, AUTO + summarize_text_detail.temperature = .25 # [0,1] + + if "" in compartment_id: + print("ERROR:Please update your compartment id in target python file") + quit() + + summarize_text_response = generative_ai_inference_client.summarize_text(summarize_text_detail) + + # Print result + #print("**************************Summarize Texts Result**************************") + print(summarize_text_response.data) + return summarize_text_response.data + + +if __name__ == '__main__': + main() \ No newline at end of file