Résumé d'actualités avec l'IA

Dans ce tutoriel, nous allons explorer un exemple simple pour résumer des actualités provenant de différentes sources et langues grâce à l’intelligence artificielle.

Nous commencerons par charger les variables d’environnement depuis un fichier .env, configurer les identifiants Azure et définir les variables d’environnement nécessaires pour utiliser les services OpenAI.

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

Fonction pour charger le contenu des URL

Cette fonction prend une URL en argument et renvoie un document propre qui pourra être traité par la suite.

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

Fonction pour obtenir les URL des flux RSS

Cette fonction extrait et renvoie les URL de chaque flux RSS fourni.

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

Fonction pour résumer le contenu des URL

Cette fonction résume le contenu des URL à l’aide du modèle de langage (LLM) d’OpenAI.

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

Code principal

Enfin, dans le bloc principal du code, nous indiquons une liste d’URL de flux RSS principaux, récupérons les URL des articles, puis générons les résumés.

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")

Code complet

Voici le code complet pour résumer des actualités de différentes sources grâce à l’intelligence artificielle :

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")