Resumen de noticias con IA

En este tutorial, exploraremos un ejemplo sencillo de cómo resumir noticias provenientes de diversas fuentes e idiomas mediante el uso de inteligencia artificial.

Comenzaremos cargando las variables de entorno de un archivo .env, configurando las credenciales de Azure, y estableciendo las variables de entorno necesarias para usar los servicios de OpenAI.

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

Función para Resumir Contenidos de URLs

Esta función toma una URL como argumento y devuelve un documento limpio que puede ser procesado más adelante.

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

Función para Obtener URLs de RSS Feeds

Esta función extrae y devuelve las URLs de cada feed RSS proporcionado.

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

Función para Resumir Contenidos de las URLs

Esta función resume los contenidos de las URLs utilizando el modelo de lenguaje (LLM) de OpenAI.

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

Código Principal

Finalmente, en el bloque principal del código, especificamos una lista de URLs de feeds RSS principales, obtenemos las URLs de las noticias y luego generamos los resúmenes.

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")

Código Completo

Aquí está el código completo para resumir noticias de diferentes fuentes utilizando inteligencia artificial:

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")