"""Crawl every page of a quotes site politely and save the results as CSV.

Settings come from the environment:
  BASE_URL        site to crawl (default https://quotes.toscrape.com)
  DELAY_SECONDS   pause between requests (default 1)
  PROXY_URL       optional, scheme://host:port of an authenticated proxy
  PROXY_USER, PROXY_PASSWORD   credentials for that proxy
"""
import csv
import os
import sys
import time
from urllib.parse import quote, urljoin, urlsplit
from urllib.robotparser import RobotFileParser

import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

BASE = os.environ.get("BASE_URL", "https://quotes.toscrape.com")
AGENT = "quotes-tutorial/1.0 (contact: you@example.com)"
DELAY = float(os.environ.get("DELAY_SECONDS", "1"))


def proxies_from_env():
    url = os.environ.get("PROXY_URL")
    if not url:
        return None
    parts = urlsplit(url)
    user = quote(os.environ["PROXY_USER"], safe="")
    password = quote(os.environ["PROXY_PASSWORD"], safe="")
    full = f"{parts.scheme}://{user}:{password}@{parts.hostname}:{parts.port}"
    return {"http": full, "https": full}


def make_session():
    retry = Retry(
        total=4,
        backoff_factor=1.0,
        status_forcelist=(429, 500, 502, 503, 504),
        allowed_methods=("GET", "HEAD"),
        respect_retry_after_header=True,
    )
    session = requests.Session()
    session.headers["User-Agent"] = AGENT
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("https://", adapter)
    session.mount("http://", adapter)
    session.proxies = proxies_from_env() or {}
    session.trust_env = False  # ignore HTTP_PROXY and similar variables
    return session


def load_robots(session):
    robots = RobotFileParser()
    response = session.get(urljoin(BASE, "/robots.txt"), timeout=(5, 20))
    if response.status_code == 407:
        response.raise_for_status()  # the proxy rejected our credentials: stop, do not guess
    if response.status_code >= 500:
        robots.disallow_all = True  # RFC 9309: treat an unreachable file as full disallow
    elif response.status_code >= 400:
        robots.allow_all = True  # no robots.txt: nothing is disallowed
    else:
        robots.parse(response.text.splitlines())
    return robots


def crawl(session, robots):
    url = urljoin(BASE, "/page/1/")
    while url:
        if not robots.can_fetch(AGENT, url):
            print(f"skipped by robots.txt: {url}", file=sys.stderr)
            return
        response = session.get(url, timeout=(5, 20))
        response.raise_for_status()
        soup = BeautifulSoup(response.text, "html.parser")
        for card in soup.select("div.quote"):
            yield {
                "text": card.select_one("span.text").get_text(strip=True),
                "author": card.select_one("small.author").get_text(strip=True),
                "tags": ";".join(t.get_text(strip=True) for t in card.select("a.tag")),
                "page": url,
            }
        link = soup.select_one("li.next a")
        url = urljoin(url, link["href"]) if link else None
        time.sleep(robots.crawl_delay(AGENT) or DELAY)


def main():
    session = make_session()
    rows = list(crawl(session, load_robots(session)))
    with open("quotes.csv", "w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=["text", "author", "tags", "page"])
        writer.writeheader()
        writer.writerows(rows)
    pages = len({row["page"] for row in rows})
    print(f"saved {len(rows)} quotes from {pages} pages")


if __name__ == "__main__":
    try:
        main()
    except requests.RequestException as error:
        # Do not print the exception text: it can contain the proxy URL with credentials.
        print(f"request failed: {type(error).__name__}", file=sys.stderr)
        sys.exit(1)