Resumo de notícias com IA

Neste tutorial, vamos explorar um exemplo simples de como resumir notícias de diversas fontes e idiomas usando inteligência artificial.

Começaremos carregando as variáveis de ambiente de um arquivo .env, configurando as credenciais do Azure e definindo as variáveis de ambiente necessárias para usar os serviços da OpenAI.

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

Função para carregar o conteúdo das URLs

Esta função recebe uma URL como argumento e devolve um documento limpo que pode ser processado depois.

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

Função para obter URLs de feeds RSS

Esta função extrai e devolve as URLs de cada feed RSS informado.

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

Função para resumir o conteúdo das URLs

Esta função resume o conteúdo das URLs usando o modelo de linguagem (LLM) da OpenAI.

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

Código principal

Por fim, no bloco principal do código, especificamos uma lista de URLs dos principais feeds RSS, obtemos as URLs das notícias e depois geramos os resumos.

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")

Código completo

Aqui está o código completo para resumir notícias de diferentes fontes usando inteligência artificial:

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")