Hello Mortar,
17 Aug 26 09:40, you wrote to me:
Is this OK with ARRL? Site owners can get rather touchy about
scraping.
From the bottom of each ARRL Letter:
"Copyright (C) 2026 American Radio Relay League, Incorporated. Use and distribution of this publication, or any portion thereof, is permitted for non-commercial or educational purposes, with attribution. All other purposes require written permission."
I make sure that's left in every "letter" I scrape.
Can you post the script? I'm interested in how that's done.
=== Cut ===
#!/usr/bin/env python3
### ARRL Letter Posting Script
###
### By Sean Dennis with assistance
### from Microsoft Copilot
###
### (C) Sean Dennis KS4TD
###
### Released under the MIT License.
import requests
import re
import textwrap
import unicodedata
import os
import sys
import time
BASE = "
https://www.arrl.org"
LETTER_LIST = "
https://www.arrl.org/arrlletter"
CACHE_FILE = os.path.expanduser("~/.arrlletter_cache")
# ------------------------------------------------------------
# Retry-capable fetcher with browser headers
# ------------------------------------------------------------
def fetch_with_retry(url, retries=5, delay=3):
headers = {
"User-Agent": (
"Mozilla/5.0 (X11; Linux x86_64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0 Safari/537.36"
)
}
for attempt in range(1, retries + 1):
try:
print("Fetching {} (attempt {}/{})".format(url, attempt, retries))
r = requests.get(url, headers=headers, timeout=20)
r.raise_for_status()
return r.text
except Exception as e:
print("Attempt {} failed: {}".format(attempt, e))
time.sleep(delay)
print("ERROR: All attempts failed.")
return None
# ------------------------------------------------------------
# Utility functions
# ------------------------------------------------------------
def log(msg):
print(msg)
def to_ascii(s):
return unicodedata.normalize("NFKD", s).encode("ascii", "ignore").decode("ascii")
def load_cached_issue():
if not os.path.exists(CACHE_FILE):
return None
try:
with open(CACHE_FILE, "r") as f:
return f.read().strip()
except:
return None
def save_cached_issue(issue):
try:
with open(CACHE_FILE, "w") as f:
f.write(issue)
except Exception as e:
log("Warning: could not save cache: {}".format(e))
# ------------------------------------------------------------
# 1. Fetch ARRL Letter index page (with retry)
# ------------------------------------------------------------
index_html = fetch_with_retry(LETTER_LIST)
if index_html is None:
sys.exit(1)
# ------------------------------------------------------------
# 2. Find newest issue link (single or double quotes)
# ------------------------------------------------------------
issues = re.findall(r"href=['\"](/arrlletterissue\?issue=\d{4}-\d{2}-\d{2})['\"]", index_html)
if not issues:
log("ERROR: No ARRL Letter issues found.")
sys.exit(1)
latest_issue_path = issues[0]
latest_issue_url = BASE + latest_issue_path
latest_issue_id = latest_issue_path.split("=")[-1]
log("Latest ARRL Letter issue: {}".format(latest_issue_id))
# ------------------------------------------------------------
# 3. Check cache
# ------------------------------------------------------------
cached = load_cached_issue()
if cached == latest_issue_id:
log("Cached issue matches latest. Nothing new to download.")
sys.exit(0)
# ------------------------------------------------------------
# 4. Download the issue HTML (with retry)
# ------------------------------------------------------------
page_html = fetch_with_retry(latest_issue_url)
if page_html is None:
sys.exit(1)
# ------------------------------------------------------------
# 5. Improved HTML cleanup
# ------------------------------------------------------------
import html
# Remove scripts and styles
clean = re.sub(r"<script.*?>.*?</script>", "", page_html, flags=re.DOTALL) clean = re.sub(r"<style.*?>.*?</style>", "", clean, flags=re.DOTALL)
# Remove all HTML tags
clean = re.sub(r"<[^>]+>", "", clean)
# Decode HTML entities ( , ', etc.)
clean = html.unescape(clean)
# Remove "undefined" junk lines
clean = clean.replace("undefined", "")
# Collapse multiple spaces
clean = re.sub(r"[ \t]+", " ", clean)
# Remove repeated blank lines
clean = re.sub(r"\n\s*\n\s*\n+", "\n\n", clean)
# Remove repeated photo captions (ARRL duplicates them)
clean = re.sub(r" \[Photo.*?\] ", "", clean)
# Remove ARRL navigation/header lines
clean = clean.replace("ARRL Home Page", "")
clean = clean.replace("ARRL Audio News", "")
clean = clean.replace("ARRL Letter Archive", "")
# Remove unsubscribe footer
clean = clean.replace("Unsubscribe from this list.", "")
# Clean up leftover blank lines
clean = re.sub(r"\n\s*\n+", "\n\n", clean)
# Normalize line endings
clean = clean.strip()
# Normalize line endings
clean = clean.strip()
# ------------------------------------------------------------
# 6. Convert to ASCII
# ------------------------------------------------------------
ascii_text = to_ascii(clean)
1# ------------------------------------------------------------
# 7. Normalize whitespace and wrap at 72 columns
# ------------------------------------------------------------
out_lines = []
for line in ascii_text.splitlines():
line = line.strip()
if not line:
out_lines.append("")
continue
wrapped = textwrap.wrap(line, width=72)
out_lines.extend(wrapped)
# ------------------------------------------------------------
# 8. Write final file
# ------------------------------------------------------------
output_file = "/tmp/arrlletter-latest.asc"
try:
with open(output_file, "w") as f:
f.write("\n".join(out_lines))
log("Created {}".format(output_file))
save_cached_issue(latest_issue_id)
except Exception as e:
log("ERROR: Could not write output file: {}".format(e))
sys.exit(1)
# ------------------------------------------------------------
=== Cut ===
Here's the actual script called by MBSE (note that the "mbmsg" lines are wrapped):
=== Cut ===
#!/bin/bash
# Environment
# PATH needed since cron doesn't inherit bash's environment
export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
# Required by MBSE
export MBSE_ROOT=/opt/mbse
$MBSE_ROOT/bin/arrl_letter.py
# Post in MIN_HAM
mbmsg post "ARS KS4TD" "All" 61 "The ARRL Letter" "/tmp/arrlletter-latest.txt"
# Post in Fidonet's HAM
mbmsg post "ARS KS4TD" "All" 117 "The ARRL Letter" "/tmp/arrlletter-latest.txt" -
=== Cut ===
I believe the Python script will run under Windows with minor modifications.
-- Sean
... "I have a love interest in every one of my films: a gun." - Schwarzenegger --- GoldED+/LNX 1.1.5-b20260304
* Origin: Outpost BBS * Johnson City, TN (1:18/200)