Initial commit

crawler/__main__.py

@@ -0,0 +1,38 @@
+# __main__.py
+import sys
+import pandas as pd
+import time
+from crawler.scraper import fetch_and_parse
+from crawler.utils import log
+
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: python -m crawler <urls.txt> <output.xlsx>")
+        sys.exit(1)
+
+    url_file = sys.argv[1]
+    output_file = sys.argv[2]
+
+    log.info(f"Lade URL-Liste aus {url_file}")
+
+    with open(url_file, "r", encoding="utf-8") as f:
+        urls = [line.strip() for line in f if line.strip()]
+
+    results = []
+
+    for url in urls:
+        try:
+            data = fetch_and_parse(url)
+            results.append(data)
+        except Exception as e:
+            log.error(f"Fehler bei {url}: {e}")
+        time.sleep(1)
+
+    df = pd.DataFrame(results)
+    df.to_excel(output_file, index=False)
+
+    log.info(f"Fertig! Datei gespeichert: {output_file}")
+
+if __name__ == "__main__":
+    main()

crawler/extractor.py

@@ -0,0 +1,113 @@
+# extractor.py
+import re
+import string
+from bs4 import BeautifulSoup
+
+STOP_WORDS = [
+    "Impressum",
+    "Datenschutz",
+    "Privacy",
+    "AGB",
+    "Terms",
+    "Kontakt",
+    "Kontaktformular"
+]
+
+def clean_printable(text):
+    printable = set(string.printable + "äöüÄÖÜ߀–—„“‚‘…")
+    return "".join(ch for ch in text if ch in printable)
+
+
+# ---------------------------------------------------------
+# MAIL + WEBSITE NUR aus exhibitors_profile_grid_pos4
+# ---------------------------------------------------------
+def extract_mailto_and_website_html(html):
+    soup = BeautifulSoup(html, "html.parser")
+
+    mailto = ""
+    website = ""
+
+    contact_li = soup.find("li", class_="exhibitors_profile_grid_pos4")
+    if not contact_li:
+        return mailto, website
+
+    for a in contact_li.find_all("a", href=True):
+        href = a["href"].strip()
+
+        if href.startswith("mailto:"):
+            mailto = href.replace("mailto:", "").strip()
+
+        elif href.startswith("http"):
+            website = href.strip()
+
+    return mailto, website
+
+
+# ---------------------------------------------------------
+# PROFIL NUR aus <h3>Unternehmensprofil</h3> + folgendem <p>
+# ---------------------------------------------------------
+def extract_profile_from_html(html):
+    soup = BeautifulSoup(html, "html.parser")
+
+    h3 = soup.find("h3", string=lambda t: t and "Unternehmensprofil" in t)
+    if not h3:
+        return ""
+
+    p = h3.find_next("p")
+    if not p:
+        return ""
+
+    # <br> → Zeilenumbrüche
+    for br in p.find_all("br"):
+        br.replace_with("\n")
+
+    text = p.get_text("\n").strip()
+    return clean_printable(text)
+
+
+# ---------------------------------------------------------
+# HAUPTFUNKTION
+# ---------------------------------------------------------
+def extract_structured_data(html, url):
+    soup = BeautifulSoup(html, "html.parser")
+    text = soup.get_text("\n")
+
+    lines = [l.strip() for l in text.split("\n") if l.strip()]
+
+    name = street = ort = kontakt = mail = telefon = ""
+    mailto = website = ""
+    profil = ""
+
+    # --- Kontaktdaten ---
+    try:
+        idx = lines.index("Kontaktdaten")
+
+        name = lines[idx + 1]
+        street = lines[idx + 2]
+        ort = lines[idx + 3]
+
+        kontakt = lines[idx + 5]
+        mail = lines[idx + 6]
+        telefon = lines[idx + 7]
+
+    except Exception as e:
+        raise ValueError(f"Kontaktdaten unvollständig: {e}")
+
+    # --- Mailto + Website aus HTML ---
+    mailto, website = extract_mailto_and_website_html(html)
+
+    # --- Profil aus HTML ---
+    profil = extract_profile_from_html(html)
+
+    return {
+        "Link": url,          # nur Text
+        "Name": name,
+        "Straße": street,
+        "Ort": ort,
+        "Kontakt": kontakt,
+        "Mail": mail,
+        "Telefon": telefon,
+        "mailto": mailto,     # nur Text
+        "Website": website,   # nur Text
+        "Profil": profil
+    }

crawler/scraper.py

@@ -0,0 +1,28 @@
+# scraper.py
+import requests
+from bs4 import BeautifulSoup
+
+from crawler.extractor import extract_structured_data
+from crawler.utils import log
+
+
+def fetch_and_parse(url):
+    log.info(f"Abruf: {url}")
+
+    headers = {"User-Agent": "Mozilla/5.0"}
+
+    try:
+        r = requests.get(url, headers=headers, timeout=10)
+        r.raise_for_status()
+    except Exception as e:
+        log.error(f"Fehler beim Abruf von {url}: {e}")
+        raise
+
+    # WICHTIG: HTML an Extractor übergeben, NICHT Text!
+    html = r.text
+
+    try:
+        return extract_structured_data(html, url)
+    except Exception as e:
+        log.error(f"Extraktionsfehler bei {url}: {e}")
+        raise

crawler/utils.py

@@ -0,0 +1,25 @@
+# utils.py
+import logging
+
+def setup_logging():
+    log = logging.getLogger("crawler")
+    log.setLevel(logging.INFO)
+
+    # Console
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+
+    # File
+    fh = logging.FileHandler("crawler.log", encoding="utf-8")
+    fh.setLevel(logging.INFO)
+
+    fmt = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+    ch.setFormatter(fmt)
+    fh.setFormatter(fmt)
+
+    log.addHandler(ch)
+    log.addHandler(fh)
+
+    return log
+
+log = setup_logging()