Riassunto di notizie con l'IA

In questo tutorial esploreremo un semplice esempio di come riassumere notizie provenienti da diverse fonti e lingue usando l’intelligenza artificiale.

Inizieremo caricando le variabili d’ambiente da un file .env, configurando le credenziali di Azure e impostando le variabili d’ambiente necessarie per usare i servizi di OpenAI.

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

Funzione per caricare il contenuto degli URL

Questa funzione riceve un URL come argomento e restituisce un documento pulito che può essere elaborato in seguito.

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

Funzione per ottenere gli URL dai feed RSS

Questa funzione estrae e restituisce gli URL di ogni feed RSS fornito.

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

Funzione per riassumere il contenuto degli URL

Questa funzione riassume il contenuto degli URL usando il modello linguistico (LLM) di OpenAI.

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

Codice principale

Infine, nel blocco principale del codice, indichiamo un elenco di URL dei principali feed RSS, otteniamo gli URL delle notizie e poi generiamo i riassunti.

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")

Codice completo

Ecco il codice completo per riassumere notizie da fonti diverse usando l’intelligenza artificiale:

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")