AI-Powered Web Scraping

import os
import importlib

# Scraper library is imported from local file
import scraper as scraper 

from dotenv import load_dotenv
from openai import OpenAI
from IPython.display import Markdown, display

# Load environment variables from a .env file
load_dotenv(override=True)
api_key = os.getenv('OPENAI_API_KEY')


# Check the key
if not api_key:
    print("No API key was found - please head over to the troubleshooting notebook in this folder to identify & fix!")
elif not api_key.startswith("sk-proj-"):
    print("An API key was found, but it doesn't start sk-proj-; please check you're using the right key - see troubleshooting notebook")
elif api_key.strip() != api_key:
    print("An API key was found, but it looks like it might have space or tab characters at the start or end - please remove them - see troubleshooting notebook")
else:
    print("API key found and looks good so far!")

from bs4 import BeautifulSoup
import requests


# Standar headers to mimic a real browser request
HEADERS = {
    # Windows
    # "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"

    #Linux
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
}

# Cache URL 
URL = None

# Store page content to avoid repeated fetches
SOUP = None

# Cache page title
PAGE_TITLE = None



def __fetch_website_contents(url):
    """
    Load the content of the website at the given url.
    """

    # Declare globals to update them
    global URL, SOUP, PAGE_TITLE


    # Retrieve the webpage content
    response = requests.get(url, headers=HEADERS)

    # Raise an error for bad responses
    response.raise_for_status()

    # Parse the HTML content using BeautifulSoup
    SOUP = BeautifulSoup(response.content, "html.parser")

    # Update new url
    URL = url

    # Cache the page title
    PAGE_TITLE = SOUP.title.string if SOUP.title else "No Title Found"

    

def fetch_text_contents(url):
    """
    Return the title and contents of the website at the given url,
    truncate to 2,000 characters as a sensible limit
    """

    if SOUP is None or URL != url:
        __fetch_website_contents(url)
    
    # Read all text content from the webpage
    if SOUP and SOUP.body:
        # Remove unwanted contents
        for irrelevant in SOUP(['script', 'style', 'img', 'input']):
            irrelevant.decompose();
        content = SOUP.body.get_text(separator='\n', strip=True)
    else:
        content = "No Body Content Found"

    return (PAGE_TITLE + "\n\n" + content)[:2000]



def fetch_website_links(url):
    """
    Return all the links found on the webpage.
    """

    if SOUP is None or URL != url:
        __fetch_website_contents(url)

    if SOUP and SOUP.body:
        links = []
        for a_tag in SOUP.find_all('a', href=True):
            links.append(a_tag['href'])
        
        return links
    else:
        return []

# Example for https://edwarddonner.com
response = scraper.fetch_text_contents("https://edwarddonner.com")
print(response)

# Example for https://edwarddonner.com
response = scraper.fetch_website_links("https://edwarddonner.com")

i = 1
for link in response:
    print(i, end=". ")
    print(link, end="\n")
    i += 1

# System Prompt
SYSTEM_PROMPT = """
    You are a snarky assistant that analyses the contents of website, 
    and provides a short, snarky, humorous summary, ignoring text that might be navigation related.
    Respond in markdown format.
    Do not wrap the makdown in code blocks. - respond just with the markdown text. 
"""

# User Prompt Prefix
user_prompt_prefix = """
    Here are the contents of a website.
    Provide a short summary of this website.
    If it includes news or announcements, then summarize these too.
"""

# Function to construct the full payload for the API call
def construct_payload(website_content):
    return [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role":"user", "content": user_prompt_prefix + website_content}
    ]

ollama_base_url = "http://localhost:11434/v1"
# If doesn't work try chaning to "http://localhost:11434".

# Initialize OpenAI client to connect to Ollama
client = OpenAI(base_url = ollama_base_url)

# Specify the local model to use
llama3 = "llama3.2:3b" 

def summarize(url):
    # 1. Fetch website content
    website_content = scraper.fetch_text_contents(url)

    # 2. Construct payload
    payload = construct_payload(website_content)

    # 3. Call the chat completions endpoint
    response = client.chat.completions.create(
        model= llama3,
        messages=payload
    )

    # 4. Extract and return the summary
    return response.choices[0].message.content

# Helper function to display the result nicely in a notebook (using IPython.display)
def display_summary(url):
    display(Markdown(summarize(url)))

display_summary("https://edwarddonner.com")

display_summary("https://cnn.com")