fix(jenkins): isolate per-feed state/cooldown, cap posts, stagger batches
- Pass --state-path and --cooldown-path per feed so a rate limit on one feed no longer triggers a global cooldown that blocks all other feeds - Add --max-posts 5 cap to prevent burst posting (e.g. 22 posts in one run) that was causing HTTP 429 errors on eurosky.social - Add 15s sleep between batches to reduce cumulative API pressure - Increase MAX_PARALLEL_FEEDS from 4 to 6 now that cooldowns are isolated - Add MAX_POSTS_PER_FEED env var for central control of the post cap - Fix missing line continuation backslash on --service argument Fixes: batch 2-7 feeds being skipped due to shared cooldown state
This commit is contained in:
28
rss2bsky.py
28
rss2bsky.py
@@ -65,8 +65,8 @@ class RetryConfig:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CooldownConfig:
|
||||
default_post_cooldown_seconds: int = 3600
|
||||
default_thumb_cooldown_seconds: int = 1800
|
||||
default_post_cooldown_seconds: int = 120
|
||||
default_thumb_cooldown_seconds: int = 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -1147,16 +1147,18 @@ def login_with_backoff(
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def run_once(
|
||||
rss_feed: str,
|
||||
bsky_handle: str,
|
||||
bsky_username: str,
|
||||
bsky_password: str,
|
||||
service_url: str,
|
||||
post_langs: List[str], # ← changed from post_lang: str
|
||||
post_langs: List[str],
|
||||
state_path: str,
|
||||
cooldown_path: str,
|
||||
cfg: AppConfig
|
||||
cfg: AppConfig,
|
||||
max_posts: int = 5 # ← NEW PARAMETER
|
||||
) -> RunResult:
|
||||
if not PIL_AVAILABLE:
|
||||
logging.warning("🟡 Pillow is not installed. External card thumbnail compression is disabled.")
|
||||
@@ -1210,6 +1212,10 @@ def run_once(
|
||||
|
||||
logging.info(f"📬 {len(entries_to_post)} entries remain after duplicate filtering.")
|
||||
|
||||
# ← NEW: log the effective cap before starting the loop
|
||||
if len(entries_to_post) > max_posts:
|
||||
logging.info(f"🔢 max-posts cap is {max_posts}: will publish at most {max_posts} of {len(entries_to_post)} entries this run.")
|
||||
|
||||
if not entries_to_post:
|
||||
logging.info("ℹ️ Execution finished: no new entries to publish.")
|
||||
return RunResult(published_count=0)
|
||||
@@ -1220,6 +1226,12 @@ def run_once(
|
||||
published = 0
|
||||
|
||||
for candidate in entries_to_post:
|
||||
|
||||
# ← NEW: hard cap check at the top of every iteration
|
||||
if published >= max_posts:
|
||||
logging.info(f"🔢 === MAX POSTS REACHED === Stopping after {published} posts (limit: {max_posts}).")
|
||||
break
|
||||
|
||||
if is_global_post_cooldown_active(cooldown_path):
|
||||
reset_str = format_cooldown_until(get_global_post_cooldown_until(cooldown_path))
|
||||
logging.error(f"🛑 === BSKY POST STOPPED: GLOBAL COOLDOWN === Skipping remaining entries until {reset_str}")
|
||||
@@ -1248,7 +1260,7 @@ def run_once(
|
||||
client=client,
|
||||
text_variants=text_variants,
|
||||
embed=embed,
|
||||
post_langs=post_langs, # ← changed
|
||||
post_langs=post_langs,
|
||||
cooldown_path=cooldown_path,
|
||||
cfg=cfg
|
||||
)
|
||||
@@ -1318,6 +1330,7 @@ def main():
|
||||
)
|
||||
parser.add_argument("--state-path", default=DEFAULT_STATE_PATH, help="Path to local JSON state file")
|
||||
parser.add_argument("--cooldown-path", default=DEFAULT_COOLDOWN_STATE_PATH, help="Path to shared cooldown JSON state file")
|
||||
parser.add_argument('--max-posts', type=int, default=5, help='Max new posts to publish per run')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Parse comma-separated langs: "ca,es" → ["ca", "es"]
|
||||
@@ -1335,10 +1348,11 @@ def main():
|
||||
bsky_username=args.bsky_username,
|
||||
bsky_password=args.bsky_app_password,
|
||||
service_url=args.service,
|
||||
post_langs=post_langs, # ← changed
|
||||
post_langs=args.lang.split(","),
|
||||
state_path=args.state_path,
|
||||
cooldown_path=args.cooldown_path,
|
||||
cfg=cfg
|
||||
cfg=AppConfig(),
|
||||
max_posts=args.max_posts
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user