Python для SEO

				
					# pip install requests beautifulsoup4 pandas tldextract
import requests, pandas as pd, tldextract
from bs4 import BeautifulSoup
from urllib.parse import urljoin

headers = {"User-Agent": "Mozilla/5.0 (compatible; SEOAuditor/1.0)"}

def fetch(url):
    try:
        r = requests.get(url, headers=headers, timeout=15, allow_redirects=True)
        return r.status_code, r.text, r.url
    except requests.RequestException:
        return None, None, None

def audit(url):
    code, html, final_url = fetch(url)
    if not html:
        return {'url': url, 'status': code, 'error': 'no_response'}

    soup = BeautifulSoup(html, 'html.parser')
    title = (soup.title.string or '').strip() if soup.title else ''
    desc = ''
    md = soup.find('meta', attrs={'name':'description'})
    if md and md.get('content'):
        desc = md['content'].strip()

    h1 = ''
    h1tag = soup.find('h1')
    if h1tag:
        h1 = h1tag.get_text(' ', strip=True)

    # каноникал
    canonical = ''
    link = soup.find('link', rel='canonical')
    if link and link.get('href'):
        canonical = urljoin(final_url or url, link['href'])

    # ALT без текста
    imgs = soup.find_all('img')
    imgs_no_alt = [img.get('src','') for img in imgs if not img.get('alt')]

    # Внутренние/внешние ссылки
    ext = tldextract.extract(final_url or url)
    root = f"{ext.domain}.{ext.suffix}"
    a_tags = soup.find_all('a', href=True)
    internal, external = 0, 0
    for a in a_tags:
        href = urljoin(final_url or url, a['href'])
        e2 = tldextract.extract(href)
        root2 = f"{e2.domain}.{e2.suffix}"
        if root2 == root:
            internal += 1
        else:
            external += 1

    # Объем текста (грубая оценка)
    text = soup.get_text(' ', strip=True)
    words = len(text.split())

    return {
        'url': url,
        'final_url': final_url,
        'status': code,
        'title': title,
        'title_len': len(title),
        'description': desc,
        'desc_len': len(desc),
        'h1': h1,
        'h1_len': len(h1),
        'canonical': canonical,
        'images_total': len(imgs),
        'images_no_alt': len(imgs_no_alt),
        'links_internal': internal,
        'links_external': external,
        'word_count': words
    }

# Список URL в urls.csv (столбец url)
urls = pd.read_csv('urls.csv')['url'].dropna().unique().tolist()
rows = [audit(u) for u in urls]
df = pd.DataFrame(rows)

# Примеры триггеров качества
problems = df[
    (df['status'] != 200) |
    (df['title_len'] < 15) |
    (df['desc_len'] < 50) |
    (df['h1_len'] < 5) |
    (df['images_no_alt'] > 0) |
    (df['word_count'] < 250)
]
df.to_csv('onpage_full_audit.csv', index=False)
problems.to_csv('onpage_problems.csv', index=False)
print(problems.head())