Endpoint Mapping (Python Crawler)

#!/usr/bin/env python3
# Simple crawler to map all links before attacking
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import sys

TARGET = sys.argv[1] if len(sys.argv) > 1 else "http://<TARGET-IP>"
visited = set()
to_visit = {TARGET}
found_endpoints = []

headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/109.0"}

def crawl(url):
    try:
        r = requests.get(url, headers=headers, timeout=5, verify=False)
        soup = BeautifulSoup(r.text, 'html.parser')
        links = []
        for tag in soup.find_all(['a', 'form', 'script', 'link']):
            href = tag.get('href') or tag.get('action') or tag.get('src')
            if href:
                full = urljoin(url, href)
                if urlparse(full).netloc == urlparse(TARGET).netloc:
                    links.append(full)
        return links
    except Exception as e:
        return []

while to_visit:
    url = to_visit.pop()
    if url in visited:
        continue
    visited.add(url)
    print(f"[+] {url}")
    found_endpoints.append(url)
    for link in crawl(url):
        if link not in visited:
            to_visit.add(link)

print("\n=== ALL ENDPOINTS ===")
for ep in sorted(found_endpoints):
    print(ep)