AI News Summaries

In this tutorial, we’ll explore a simple example of how to summarise news from different sources and languages using artificial intelligence.

We’ll start by loading the environment variables from a .env file, setting up the Azure credentials, and setting the environment variables needed to use the OpenAI services.

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

Function to Load Content from URLs

This function takes a URL as its argument and returns a clean document that can be processed later.

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

Function to Get URLs from RSS Feeds

This function extracts and returns the URLs from each RSS feed provided.

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

Function to Summarise the Content of the URLs

This function summarises the content of the URLs using OpenAI’s language model (LLM).

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

Main Code

Finally, in the main block of the code, we specify a list of main RSS feed URLs, get the news URLs and then generate the summaries.

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")

Complete Code

Here is the complete code to summarise news from different sources using artificial intelligence:

import os
from azure.identity import ChainedTokenCredential, EnvironmentCredential
from langchain_openai import AzureChatOpenAI
from langchain_community.document_loaders import UnstructuredURLLoader
from unstructured.cleaners.core import remove_punctuation, clean, clean_extra_whitespace
from langchain.docstore.document import Document
import requests
from dotenv import load_dotenv
import xml.etree.ElementTree as ET

load_dotenv()

credential = ChainedTokenCredential(EnvironmentCredential())
access_token = credential.get_token("https://cognitiveservices.azure.com/.default")

os.environ.update({
    "AZURE_OPENAI_ENDPOINT": "https://zerogap.openai.azure.com/",
    "AZURE_OPENAI_API_KEY": access_token.token,
    "OPENAI_API_TYPE": "azure_ad",
    "OPENAI_DEPLOYMENT": "gpt-4o-mini"
})

llm = AzureChatOpenAI(openai_api_version="2024-08-01-preview", azure_deployment="gpt-4o-mini", temperature=0.5)

def summarize(url):
    loader = UnstructuredURLLoader(urls=[url], mode="elements", post_processors=[clean, remove_punctuation, clean_extra_whitespace])
    elements = loader.load()
    full_clean = " ".join([e.page_content for e in elements if e.metadata['category'] == "NarrativeText"])
    return Document(page_content=full_clean, metadata={"source": url})

def get_urls(main_urls):
    all_urls = {}
    for url in main_urls:
        try:
            response = requests.get(url)
            root = ET.fromstring(response.content)
            urls = [item.find("link").text for item in root.findall(".//item") if item.find("link").text]
            all_urls[url] = urls
        except Exception:
            continue
    return all_urls

def summarize_urls(urls_by_source):
    summaries_by_source = {}
    for source, urls in urls_by_source.items():
        summaries = []
        for url in urls:
            doc = summarize(url)
            response = llm.invoke(f"Create a short summary in English for the following content:\n\n{doc.page_content}")
            summaries.append(response.content)
        summaries_by_source[source] = summaries
    return summaries_by_source

if __name__ == "__main__":
    main_urls = [
        "https://feeds.bbci.co.uk/news/business/economy/rss.xml?edition=uk",
        "https://feeds.elpais.com/mrss-s/pages/ep/site/elpais.com/section/economia/portada"
    ]
    urls_by_source = get_urls(main_urls)
    print("Summarizing URLs...")
    summaries_by_source = summarize_urls(urls_by_source)
    for source, summaries in summaries_by_source.items():
        print(f"Source: {source}")
        for i, summary in enumerate(summaries, 1):
            print(f"\n{summary}\n")
        print("\n")