fix(sync): preserve non-X links during truncation so Bluesky external cards are generated

twitter2bsky_daemon.py

@@ -35,14 +35,13 @@ EXTERNAL_THUMB_MIN_JPEG_QUALITY = 40
 BSKY_BLOB_UPLOAD_MAX_RETRIES = 5
 BSKY_BLOB_UPLOAD_BASE_DELAY = 10
 BSKY_BLOB_UPLOAD_MAX_DELAY = 300
+BSKY_BLOB_TRANSIENT_ERROR_RETRIES = 3
+BSKY_BLOB_TRANSIENT_ERROR_DELAY = 15
+
 MEDIA_DOWNLOAD_TIMEOUT = 30
 LINK_METADATA_TIMEOUT = 10
 DEFAULT_BSKY_BASE_URL = "https://bsky.social"
 
-# Extra timeout retry tuning for transient blob upload failures
-BSKY_BLOB_TRANSIENT_ERROR_RETRIES = 3
-BSKY_BLOB_TRANSIENT_ERROR_DELAY = 15
-
 # --- Logging Setup ---
 logging.basicConfig(
     format="%(asctime)s [%(levelname)s] %(message)s",
@@ -242,74 +241,295 @@ def remove_trailing_ellipsis_line(text):
     return "\n".join(lines).strip()
 
 
-def clean_url(url):
+def clean_post_text(text):
-    trimmed_url = url.strip()
+    raw_text = (text or "").strip()
-    cleaned_url = re.sub(r"\s+", "", trimmed_url)
+    raw_text = repair_broken_urls(raw_text)
-    cleaned_url = strip_trailing_url_punctuation(cleaned_url)
+    raw_text = repair_broken_mentions(raw_text)
-
+    raw_text = strip_line_edge_whitespace(raw_text)
-    if is_valid_url(cleaned_url):
+    raw_text = remove_trailing_ellipsis_line(raw_text)
-        return cleaned_url
+    return raw_text.strip()
-    return None
 
 
-def canonicalize_url(url):
+def truncate_text_safely(text, max_length=BSKY_TEXT_MAX_LENGTH):
-    if not url:
+    if len(text) <= max_length:
-        return None
+        return text
-    return strip_trailing_url_punctuation(url.strip())
+
+    truncated = text[:max_length - 3]
+    last_space = truncated.rfind(" ")
+    if last_space > 0:
+        return truncated[:last_space] + "..."
+    return truncated + "..."
 
 
-def canonicalize_tweet_url(url):
+def prepare_post_text_for_bsky(full_clean_text, keep_url=None):
-    if not url:
+    """
-        return None
+    Prepare final Bluesky post text.
-
+    If keep_url is provided and exists in the text, try to preserve it in the final output
-    url = url.strip()
+    by truncating the body before the URL instead of cutting the URL away.
-    match = re.search(r"https?://(?:www\.)?(?:x\.com|twitter\.com)/([^/]+)/status/(\d+)", url, re.IGNORECASE)
+    """
-    if not match:
+    text = (full_clean_text or "").strip()
-        return url.lower()
-
-    handle = match.group(1).lower()
-    tweet_id = match.group(2)
-    return f"https://x.com/{handle}/status/{tweet_id}"
-
-
-def is_x_or_twitter_domain(url):
-    try:
-        hostname = (urlparse(url).hostname or "").lower()
-        return hostname in {"x.com", "www.x.com", "twitter.com", "www.twitter.com", "mobile.twitter.com"}
-    except Exception:
-        return False
-
-
-def extract_urls_from_text(text):
     if not text:
-        return []
+        return text
-    repaired = repair_broken_urls(text)
-    return re.findall(r"https?://[^\s]+", repaired)
 
+    if len(text) <= BSKY_TEXT_MAX_LENGTH:
+        return text
 
-def extract_non_x_urls_from_text(text):
+    if keep_url:
-    urls = extract_urls_from_text(text)
+        canonical_keep = canonicalize_url(keep_url)
-    result = []
+        urls = extract_ordered_non_x_urls(text)
 
+        matched_url = None
         for url in urls:
-        cleaned = strip_trailing_url_punctuation(url)
+            if canonicalize_url(url) == canonical_keep:
-        if cleaned and not is_x_or_twitter_domain(cleaned):
+                matched_url = url
-            result.append(cleaned)
+                break
 
-    return result
+        if matched_url and matched_url in text:
+            idx = text.find(matched_url)
+            prefix = text[:idx].rstrip()
+            suffix = matched_url
+
+            reserve = len(suffix) + 1
+            available = BSKY_TEXT_MAX_LENGTH - reserve
+
+            if available > 10:
+                trimmed_prefix = prefix
+                if len(trimmed_prefix) > available:
+                    trimmed_prefix = trimmed_prefix[:available - 3]
+                    last_space = trimmed_prefix.rfind(" ")
+                    if last_space > 0:
+                        trimmed_prefix = trimmed_prefix[:last_space] + "..."
+                    else:
+                        trimmed_prefix = trimmed_prefix + "..."
+
+                final_text = f"{trimmed_prefix.rstrip()} {suffix}".strip()
+                if len(final_text) <= BSKY_TEXT_MAX_LENGTH:
+                    logging.info("🔗 Preserved non-X URL in final Bluesky text for card generation")
+                    return final_text
+
+    return truncate_text_safely(text, BSKY_TEXT_MAX_LENGTH)
 
 
-def extract_ordered_non_x_urls(text):
+def normalize_post_text(text):
-    seen = set()
+    if not text:
-    ordered = []
+        return ""
 
-    for url in extract_non_x_urls_from_text(text):
+    text = clean_post_text(text)
-        canonical = canonicalize_url(url)
+    text = text.replace("\r", "\n")
-        if canonical and canonical not in seen:
+    text = re.sub(r"\s+", " ", text).strip()
-            seen.add(canonical)
+    return text.lower()
-            ordered.append(canonical)
 
-    return ordered
+
+def build_media_fingerprint(tweet):
+    if not tweet or not tweet.media:
+        return "no-media"
+
+    parts = []
+
+    for media in tweet.media:
+        media_type = getattr(media, "type", "unknown")
+        media_url = getattr(media, "media_url_https", "") or ""
+
+        stable_value = media_url
+
+        if media_type == "photo":
+            stable_value = re.sub(r"[?&]name=\w+", "", stable_value)
+            stable_value = re.sub(r"[?&]format=\w+", "", stable_value)
+        elif media_type == "video":
+            stable_value = canonicalize_tweet_url(tweet.tweet_url or media_url or "")
+
+        parts.append(f"{media_type}:{stable_value}")
+
+    parts.sort()
+    raw = "|".join(parts)
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def build_bsky_media_fingerprint(post_view):
+    try:
+        embed = getattr(post_view, "embed", None)
+        if not embed:
+            return "no-media"
+
+        parts = []
+
+        images = getattr(embed, "images", None)
+        if images:
+            for img in images:
+                image_obj = getattr(img, "image", None)
+                ref = getattr(image_obj, "ref", None) or getattr(image_obj, "cid", None) or str(image_obj)
+                parts.append(f"photo:{ref}")
+
+        video = getattr(embed, "video", None)
+        if video:
+            ref = getattr(video, "ref", None) or getattr(video, "cid", None) or str(video)
+            parts.append(f"video:{ref}")
+
+        external = getattr(embed, "external", None)
+        if external:
+            uri = getattr(external, "uri", None) or str(external)
+            parts.append(f"external:{uri}")
+
+        if not parts:
+            return "no-media"
+
+        parts.sort()
+        raw = "|".join(parts)
+        return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+    except Exception as e:
+        logging.debug(f"Could not build Bluesky media fingerprint: {e}")
+        return "no-media"
+
+
+def build_text_media_key(normalized_text, media_fingerprint):
+    return hashlib.sha256(f"{normalized_text}||{media_fingerprint}".encode("utf-8")).hexdigest()
+
+
+def create_bsky_client(base_url, handle, password):
+    normalized_base_url = (base_url or DEFAULT_BSKY_BASE_URL).strip().rstrip("/")
+    logging.info(f"🔐 Connecting Bluesky client via base URL: {normalized_base_url}")
+
+    try:
+        client = Client(base_url=normalized_base_url)
+    except TypeError:
+        logging.warning("⚠️ Your atproto Client does not accept base_url in constructor. Falling back.")
+        client = Client()
+        try:
+            if hasattr(client, "base_url"):
+                client.base_url = normalized_base_url
+            elif hasattr(client, "_base_url"):
+                client._base_url = normalized_base_url
+        except Exception as e:
+            logging.warning(f"⚠️ Could not apply custom base URL cleanly: {e}")
+
+    client.login(handle, password)
+    return client
+
+
+def default_state():
+    return {
+        "version": 1,
+        "posted_tweets": {},
+        "posted_by_bsky_uri": {},
+        "updated_at": None,
+    }
+
+
+def load_state(state_path=STATE_PATH):
+    if not os.path.exists(state_path):
+        logging.info(f"🧠 No state file found at {state_path}. Starting with empty memory.")
+        return default_state()
+
+    try:
+        with open(state_path, "r", encoding="utf-8") as f:
+            state = json.load(f)
+
+        if not isinstance(state, dict):
+            logging.warning("⚠️ State file is invalid. Reinitializing.")
+            return default_state()
+
+        state.setdefault("version", 1)
+        state.setdefault("posted_tweets", {})
+        state.setdefault("posted_by_bsky_uri", {})
+        state.setdefault("updated_at", None)
+
+        return state
+
+    except Exception as e:
+        logging.warning(f"⚠️ Could not load state file {state_path}: {e}. Reinitializing.")
+        return default_state()
+
+
+def save_state(state, state_path=STATE_PATH):
+    try:
+        state["updated_at"] = arrow.utcnow().isoformat()
+        temp_path = f"{state_path}.tmp"
+
+        with open(temp_path, "w", encoding="utf-8") as f:
+            json.dump(state, f, ensure_ascii=False, indent=2, sort_keys=True)
+
+        os.replace(temp_path, state_path)
+        logging.info(f"💾 State saved to {state_path}")
+
+    except Exception as e:
+        logging.error(f"❌ Failed to save state file {state_path}: {e}")
+
+
+def remember_posted_tweet(state, candidate, bsky_uri=None):
+    canonical_tweet_url = candidate.get("canonical_tweet_url")
+    fallback_key = f"textmedia:{candidate['text_media_key']}"
+    state_key = canonical_tweet_url or fallback_key
+
+    record = {
+        "canonical_tweet_url": canonical_tweet_url,
+        "normalized_text": candidate["normalized_text"],
+        "raw_text": candidate["raw_text"],
+        "full_clean_text": candidate.get("full_clean_text", candidate["raw_text"]),
+        "media_fingerprint": candidate["media_fingerprint"],
+        "text_media_key": candidate["text_media_key"],
+        "canonical_non_x_urls": sorted(candidate["canonical_non_x_urls"]),
+        "ordered_non_x_urls": candidate.get("ordered_non_x_urls", []),
+        "bsky_uri": bsky_uri,
+        "tweet_created_on": candidate["tweet"].created_on,
+        "tweet_url": candidate["tweet"].tweet_url,
+        "posted_at": arrow.utcnow().isoformat(),
+    }
+
+    state["posted_tweets"][state_key] = record
+
+    if bsky_uri:
+        state["posted_by_bsky_uri"][bsky_uri] = state_key
+
+
+def candidate_matches_state(candidate, state):
+    canonical_tweet_url = candidate["canonical_tweet_url"]
+    text_media_key = candidate["text_media_key"]
+    normalized_text = candidate["normalized_text"]
+
+    posted_tweets = state.get("posted_tweets", {})
+
+    if canonical_tweet_url and canonical_tweet_url in posted_tweets:
+        return True, "state:tweet_url"
+
+    for _, record in posted_tweets.items():
+        if record.get("text_media_key") == text_media_key:
+            return True, "state:text_media_fingerprint"
+
+    for _, record in posted_tweets.items():
+        if record.get("normalized_text") == normalized_text:
+            return True, "state:normalized_text"
+
+    return False, None
+
+
+def prune_state(state, max_entries=5000):
+    posted_tweets = state.get("posted_tweets", {})
+
+    if len(posted_tweets) <= max_entries:
+        return state
+
+    sortable = []
+    for key, record in posted_tweets.items():
+        posted_at = record.get("posted_at") or ""
+        sortable.append((key, posted_at))
+
+    sortable.sort(key=lambda x: x[1], reverse=True)
+    keep_keys = {key for key, _ in sortable[:max_entries]}
+
+    new_posted_tweets = {}
+    for key, record in posted_tweets.items():
+        if key in keep_keys:
+            new_posted_tweets[key] = record
+
+    new_posted_by_bsky_uri = {}
+    for bsky_uri, key in state.get("posted_by_bsky_uri", {}).items():
+        if key in keep_keys:
+            new_posted_by_bsky_uri[bsky_uri] = key
+
+    state["posted_tweets"] = new_posted_tweets
+    state["posted_by_bsky_uri"] = new_posted_by_bsky_uri
+    return state
 
 
 def extract_urls_from_facets(record):
@@ -329,21 +549,55 @@ def extract_urls_from_facets(record):
     return urls
 
 
-def looks_like_title_plus_url_post(text):
+def get_recent_bsky_posts(client, handle, limit=30):
-    if not text:
+    recent_posts = []
-        return False
 
-    repaired = repair_broken_urls(text)
+    try:
-    repaired = strip_line_edge_whitespace(repaired)
+        timeline = client.get_author_feed(handle, limit=limit)
-    lines = [line.strip() for line in repaired.splitlines() if line.strip()]
-    if len(lines) < 2:
-        return False
 
-    last_line = lines[-1]
+        for item in timeline.feed:
-    urls_in_last_line = extract_ordered_non_x_urls(last_line)
+            try:
-    total_urls = extract_ordered_non_x_urls(repaired)
+                if item.reason is not None:
+                    continue
 
-    return len(urls_in_last_line) == 1 and len(total_urls) == 1 and last_line.startswith(("http://", "https://"))
+                record = item.post.record
+                if getattr(record, "reply", None) is not None:
+                    continue
+
+                text = getattr(record, "text", "") or ""
+                normalized_text = normalize_post_text(text)
+
+                urls = []
+                urls.extend(extract_non_x_urls_from_text(text))
+                urls.extend(extract_urls_from_facets(record))
+
+                canonical_non_x_urls = set()
+                for url in urls:
+                    if not is_x_or_twitter_domain(url):
+                        canonical = canonicalize_url(url)
+                        if canonical:
+                            canonical_non_x_urls.add(canonical)
+
+                media_fingerprint = build_bsky_media_fingerprint(item.post)
+                text_media_key = build_text_media_key(normalized_text, media_fingerprint)
+
+                recent_posts.append({
+                    "uri": getattr(item.post, "uri", None),
+                    "text": text,
+                    "normalized_text": normalized_text,
+                    "canonical_non_x_urls": canonical_non_x_urls,
+                    "media_fingerprint": media_fingerprint,
+                    "text_media_key": text_media_key,
+                    "created_at": getattr(record, "created_at", None),
+                })
+
+            except Exception as e:
+                logging.debug(f"Skipping one Bluesky feed item during dedupe fetch: {e}")
+
+    except Exception as e:
+        logging.warning(f"⚠️ Could not fetch recent Bluesky posts for duplicate detection: {e}")
+
+    return recent_posts
 
 
 def get_rate_limit_wait_seconds(error_obj, default_delay):
@@ -642,305 +896,9 @@ def build_external_link_embed(url, client, http_client, fallback_title="Link"):
     return None
 
 
-def prepare_post_text(text):
-    raw_text = (text or "").strip()
-    raw_text = repair_broken_urls(raw_text)
-    raw_text = repair_broken_mentions(raw_text)
-    raw_text = strip_line_edge_whitespace(raw_text)
-    raw_text = remove_trailing_ellipsis_line(raw_text)
-
-    if len(raw_text) > BSKY_TEXT_MAX_LENGTH:
-        truncated = raw_text[:BSKY_TEXT_MAX_LENGTH - 3]
-        last_space = truncated.rfind(" ")
-        if last_space > 0:
-            raw_text = truncated[:last_space] + "..."
-        else:
-            raw_text = truncated + "..."
-
-    return raw_text.strip()
-
-
-def normalize_post_text(text):
-    if not text:
-        return ""
-
-    text = repair_broken_urls(text)
-    text = repair_broken_mentions(text)
-    text = strip_line_edge_whitespace(text)
-    text = remove_trailing_ellipsis_line(text)
-    text = text.replace("\r", "\n")
-    text = re.sub(r"\s+", " ", text).strip()
-    return text.lower()
-
-
-def build_media_fingerprint(tweet):
-    if not tweet or not tweet.media:
-        return "no-media"
-
-    parts = []
-
-    for media in tweet.media:
-        media_type = getattr(media, "type", "unknown")
-        media_url = getattr(media, "media_url_https", "") or ""
-
-        stable_value = media_url
-
-        if media_type == "photo":
-            stable_value = re.sub(r"[?&]name=\w+", "", stable_value)
-            stable_value = re.sub(r"[?&]format=\w+", "", stable_value)
-        elif media_type == "video":
-            stable_value = canonicalize_tweet_url(tweet.tweet_url or media_url or "")
-
-        parts.append(f"{media_type}:{stable_value}")
-
-    parts.sort()
-    raw = "|".join(parts)
-    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
-
-
-def build_bsky_media_fingerprint(post_view):
-    try:
-        embed = getattr(post_view, "embed", None)
-        if not embed:
-            return "no-media"
-
-        parts = []
-
-        images = getattr(embed, "images", None)
-        if images:
-            for img in images:
-                image_obj = getattr(img, "image", None)
-                ref = getattr(image_obj, "ref", None) or getattr(image_obj, "cid", None) or str(image_obj)
-                parts.append(f"photo:{ref}")
-
-        video = getattr(embed, "video", None)
-        if video:
-            ref = getattr(video, "ref", None) or getattr(video, "cid", None) or str(video)
-            parts.append(f"video:{ref}")
-
-        external = getattr(embed, "external", None)
-        if external:
-            uri = getattr(external, "uri", None) or str(external)
-            parts.append(f"external:{uri}")
-
-        if not parts:
-            return "no-media"
-
-        parts.sort()
-        raw = "|".join(parts)
-        return hashlib.sha256(raw.encode("utf-8")).hexdigest()
-
-    except Exception as e:
-        logging.debug(f"Could not build Bluesky media fingerprint: {e}")
-        return "no-media"
-
-
-def build_text_media_key(normalized_text, media_fingerprint):
-    return hashlib.sha256(f"{normalized_text}||{media_fingerprint}".encode("utf-8")).hexdigest()
-
-
-def create_bsky_client(base_url, handle, password):
-    normalized_base_url = (base_url or DEFAULT_BSKY_BASE_URL).strip().rstrip("/")
-    logging.info(f"🔐 Connecting Bluesky client via base URL: {normalized_base_url}")
-
-    try:
-        client = Client(base_url=normalized_base_url)
-    except TypeError:
-        logging.warning("⚠️ Your atproto Client does not accept base_url in constructor. Falling back.")
-        client = Client()
-        try:
-            if hasattr(client, "base_url"):
-                client.base_url = normalized_base_url
-            elif hasattr(client, "_base_url"):
-                client._base_url = normalized_base_url
-        except Exception as e:
-            logging.warning(f"⚠️ Could not apply custom base URL cleanly: {e}")
-
-    client.login(handle, password)
-    return client
-
-
-def default_state():
-    return {
-        "version": 1,
-        "posted_tweets": {},
-        "posted_by_bsky_uri": {},
-        "updated_at": None,
-    }
-
-
-def load_state(state_path=STATE_PATH):
-    if not os.path.exists(state_path):
-        logging.info(f"🧠 No state file found at {state_path}. Starting with empty memory.")
-        return default_state()
-
-    try:
-        with open(state_path, "r", encoding="utf-8") as f:
-            state = json.load(f)
-
-        if not isinstance(state, dict):
-            logging.warning("⚠️ State file is invalid. Reinitializing.")
-            return default_state()
-
-        state.setdefault("version", 1)
-        state.setdefault("posted_tweets", {})
-        state.setdefault("posted_by_bsky_uri", {})
-        state.setdefault("updated_at", None)
-
-        return state
-
-    except Exception as e:
-        logging.warning(f"⚠️ Could not load state file {state_path}: {e}. Reinitializing.")
-        return default_state()
-
-
-def save_state(state, state_path=STATE_PATH):
-    try:
-        state["updated_at"] = arrow.utcnow().isoformat()
-        temp_path = f"{state_path}.tmp"
-
-        with open(temp_path, "w", encoding="utf-8") as f:
-            json.dump(state, f, ensure_ascii=False, indent=2, sort_keys=True)
-
-        os.replace(temp_path, state_path)
-        logging.info(f"💾 State saved to {state_path}")
-
-    except Exception as e:
-        logging.error(f"❌ Failed to save state file {state_path}: {e}")
-
-
-def remember_posted_tweet(state, candidate, bsky_uri=None):
-    canonical_tweet_url = candidate.get("canonical_tweet_url")
-    fallback_key = f"textmedia:{candidate['text_media_key']}"
-    state_key = canonical_tweet_url or fallback_key
-
-    record = {
-        "canonical_tweet_url": canonical_tweet_url,
-        "normalized_text": candidate["normalized_text"],
-        "raw_text": candidate["raw_text"],
-        "media_fingerprint": candidate["media_fingerprint"],
-        "text_media_key": candidate["text_media_key"],
-        "canonical_non_x_urls": sorted(candidate["canonical_non_x_urls"]),
-        "ordered_non_x_urls": candidate.get("ordered_non_x_urls", []),
-        "bsky_uri": bsky_uri,
-        "tweet_created_on": candidate["tweet"].created_on,
-        "tweet_url": candidate["tweet"].tweet_url,
-        "posted_at": arrow.utcnow().isoformat(),
-    }
-
-    state["posted_tweets"][state_key] = record
-
-    if bsky_uri:
-        state["posted_by_bsky_uri"][bsky_uri] = state_key
-
-
-def candidate_matches_state(candidate, state):
-    canonical_tweet_url = candidate["canonical_tweet_url"]
-    text_media_key = candidate["text_media_key"]
-    normalized_text = candidate["normalized_text"]
-
-    posted_tweets = state.get("posted_tweets", {})
-
-    if canonical_tweet_url and canonical_tweet_url in posted_tweets:
-        return True, "state:tweet_url"
-
-    for _, record in posted_tweets.items():
-        if record.get("text_media_key") == text_media_key:
-            return True, "state:text_media_fingerprint"
-
-    for _, record in posted_tweets.items():
-        if record.get("normalized_text") == normalized_text:
-            return True, "state:normalized_text"
-
-    return False, None
-
-
-def prune_state(state, max_entries=5000):
-    posted_tweets = state.get("posted_tweets", {})
-
-    if len(posted_tweets) <= max_entries:
-        return state
-
-    sortable = []
-    for key, record in posted_tweets.items():
-        posted_at = record.get("posted_at") or ""
-        sortable.append((key, posted_at))
-
-    sortable.sort(key=lambda x: x[1], reverse=True)
-    keep_keys = {key for key, _ in sortable[:max_entries]}
-
-    new_posted_tweets = {}
-    for key, record in posted_tweets.items():
-        if key in keep_keys:
-            new_posted_tweets[key] = record
-
-    new_posted_by_bsky_uri = {}
-    for bsky_uri, key in state.get("posted_by_bsky_uri", {}).items():
-        if key in keep_keys:
-            new_posted_by_bsky_uri[bsky_uri] = key
-
-    state["posted_tweets"] = new_posted_tweets
-    state["posted_by_bsky_uri"] = new_posted_by_bsky_uri
-    return state
-
-
-def get_recent_bsky_posts(client, handle, limit=30):
-    recent_posts = []
-
-    try:
-        timeline = client.get_author_feed(handle, limit=limit)
-
-        for item in timeline.feed:
-            try:
-                if item.reason is not None:
-                    continue
-
-                record = item.post.record
-                if getattr(record, "reply", None) is not None:
-                    continue
-
-                text = getattr(record, "text", "") or ""
-                normalized_text = normalize_post_text(text)
-
-                urls = []
-                urls.extend(extract_non_x_urls_from_text(text))
-                urls.extend(extract_urls_from_facets(record))
-
-                canonical_non_x_urls = set()
-                for url in urls:
-                    if not is_x_or_twitter_domain(url):
-                        canonical = canonicalize_url(url)
-                        if canonical:
-                            canonical_non_x_urls.add(canonical)
-
-                media_fingerprint = build_bsky_media_fingerprint(item.post)
-                text_media_key = build_text_media_key(normalized_text, media_fingerprint)
-
-                recent_posts.append({
-                    "uri": getattr(item.post, "uri", None),
-                    "text": text,
-                    "normalized_text": normalized_text,
-                    "canonical_non_x_urls": canonical_non_x_urls,
-                    "media_fingerprint": media_fingerprint,
-                    "text_media_key": text_media_key,
-                    "created_at": getattr(record, "created_at", None),
-                })
-
-            except Exception as e:
-                logging.debug(f"Skipping one Bluesky feed item during dedupe fetch: {e}")
-
-    except Exception as e:
-        logging.warning(f"⚠️ Could not fetch recent Bluesky posts for duplicate detection: {e}")
-
-    return recent_posts
-
-
 def make_rich(content):
     text_builder = client_utils.TextBuilder()
-    content = repair_broken_urls(content.strip())
+    content = clean_post_text(content)
-    content = repair_broken_mentions(content)
-    content = strip_line_edge_whitespace(content)
-    content = remove_trailing_ellipsis_line(content)
     lines = content.splitlines()
 
     for line_idx, line in enumerate(lines):
@@ -985,10 +943,7 @@ def make_rich(content):
 
 
 def build_dynamic_alt(raw_text):
-    dynamic_alt = repair_broken_urls(raw_text)
+    dynamic_alt = clean_post_text(raw_text)
-    dynamic_alt = repair_broken_mentions(dynamic_alt)
-    dynamic_alt = strip_line_edge_whitespace(dynamic_alt)
-    dynamic_alt = remove_trailing_ellipsis_line(dynamic_alt)
     dynamic_alt = dynamic_alt.replace("\n", " ").strip()
     dynamic_alt = re.sub(r"https?://\S+", "", dynamic_alt).strip()
 
@@ -1469,33 +1424,38 @@ def sync_feeds(args):
                     logging.info(f"⏭️ Skipping old tweet from {tweet_time}")
                     continue
 
-                prepared_text = prepare_post_text(tweet.text)
+                full_clean_text = clean_post_text(tweet.text)
-                normalized_text = normalize_post_text(prepared_text)
+                normalized_text = normalize_post_text(full_clean_text)
 
                 if not normalized_text:
                     logging.info(f"⏭️ Skipping empty/blank tweet from {tweet_time}")
                     continue
 
+                ordered_non_x_urls = extract_ordered_non_x_urls(full_clean_text)
+                canonical_non_x_urls = set(ordered_non_x_urls)
+                primary_non_x_url = ordered_non_x_urls[0] if ordered_non_x_urls else None
+
+                raw_text = prepare_post_text_for_bsky(full_clean_text, keep_url=primary_non_x_url)
+
                 media_fingerprint = build_media_fingerprint(tweet)
                 text_media_key = build_text_media_key(normalized_text, media_fingerprint)
 
-                ordered_non_x_urls = extract_ordered_non_x_urls(prepared_text)
-                canonical_non_x_urls = set(ordered_non_x_urls)
-
                 has_video = any(getattr(m, "type", None) == "video" for m in (tweet.media or []))
                 has_photo = any(getattr(m, "type", None) == "photo" for m in (tweet.media or []))
 
                 candidate_tweets.append({
                     "tweet": tweet,
                     "tweet_time": tweet_time,
-                    "raw_text": prepared_text,
+                    "raw_text": raw_text,
+                    "full_clean_text": full_clean_text,
                     "normalized_text": normalized_text,
                     "media_fingerprint": media_fingerprint,
                     "text_media_key": text_media_key,
                     "canonical_tweet_url": canonicalize_tweet_url(tweet.tweet_url),
                     "canonical_non_x_urls": canonical_non_x_urls,
                     "ordered_non_x_urls": ordered_non_x_urls,
-                    "looks_like_title_plus_url": looks_like_title_plus_url_post(prepared_text),
+                    "primary_non_x_url": primary_non_x_url,
+                    "looks_like_title_plus_url": looks_like_title_plus_url_post(full_clean_text),
                     "has_video": has_video,
                     "has_photo": has_photo,
                 })
@@ -1550,11 +1510,12 @@ def sync_feeds(args):
                 tweet = candidate["tweet"]
                 tweet_time = candidate["tweet_time"]
                 raw_text = candidate["raw_text"]
+                full_clean_text = candidate["full_clean_text"]
 
                 logging.info(f"📝 Posting missing tweet from {tweet_time} to Bluesky...")
 
                 rich_text = make_rich(raw_text)
-                dynamic_alt = build_dynamic_alt(raw_text)
+                dynamic_alt = build_dynamic_alt(full_clean_text)
 
                 image_embeds = []
                 video_embed = None
@@ -1563,9 +1524,6 @@ def sync_feeds(args):
 
                 has_video = candidate.get("has_video", False)
 
-                # --- VIDEO-FIRST POLICY ---
-                # If the tweet contains video, try video first and do not degrade to photos
-                # from the same tweet if video processing/upload fails.
                 if has_video:
                     video_media = next((m for m in (tweet.media or []) if getattr(m, "type", None) == "video"), None)
 
@@ -1598,7 +1556,6 @@ def sync_feeds(args):
                                 if os.path.exists(temp_video_path):
                                     os.remove(temp_video_path)
 
-                    # Important: if tweet had video, do NOT upload photos as fallback.
                     if not video_embed:
                         logging.warning(
                             "⚠️ Tweet contains video, but video could not be posted. "
@@ -1606,7 +1563,6 @@ def sync_feeds(args):
                         )
 
                 else:
-                    # Photo-only tweets can post images normally.
                     if tweet.media:
                         for media in tweet.media:
                             if media.type == "photo":
@@ -1621,19 +1577,15 @@ def sync_feeds(args):
                                 else:
                                     media_upload_failures.append(f"photo:{media.media_url_https}")
 
-                # If nothing media-based is available, optionally degrade to external card / text-only
                 if not video_embed and not image_embeds:
-                    candidate_url = None
+                    candidate_url = candidate.get("primary_non_x_url")
-
-                    if candidate.get("ordered_non_x_urls"):
-                        candidate_url = candidate["ordered_non_x_urls"][0]
 
+                    if candidate_url:
                         if candidate.get("looks_like_title_plus_url"):
                             logging.info(f"🔗 Detected title+URL post style. Using URL for external card: {candidate_url}")
                         else:
-                            logging.info(f"🔗 Text-only post with non-X URL. Using first URL for external card: {candidate_url}")
+                            logging.info(f"🔗 Using first non-X URL for external card: {candidate_url}")
 
-                    if candidate_url:
                         external_embed = build_external_link_embed(
                             candidate_url,
                             bsky_client,