diff --git a/jenkins/ePrss b/jenkins/ePrss index c1ea140..72f7b4d 100644 --- a/jenkins/ePrss +++ b/jenkins/ePrss @@ -8,20 +8,20 @@ pipeline { disableConcurrentBuilds() } -triggers { - cron(''' + triggers { + cron(''' H 22,10,16 * * * 0 6 * * * ''') -} - + } environment { - VENV_DIR = 'venv' - MAX_PARALLEL_FEEDS = '4' + VENV_DIR = 'venv' + MAX_PARALLEL_FEEDS = '6' JITTER_MAX_SECONDS = '12' - PYTHONUNBUFFERED = '1' - PIP_CACHE_DIR = "${WORKSPACE}/.pip-cache" + MAX_POSTS_PER_FEED = '5' + PYTHONUNBUFFERED = '1' + PIP_CACHE_DIR = "${WORKSPACE}/.pip-cache" } stages { @@ -49,9 +49,9 @@ H 22,10,16 * * * stage('Process All RSS Feeds (Batched Parallel)') { steps { withCredentials([ - string(credentialsId: 'BSKY_EP_HANDLE', variable: 'BSKY_EP_HANDLE'), - string(credentialsId: 'BSKY_EP_USERNAME', variable: 'BSKY_EP_USERNAME'), - string(credentialsId: 'BSKY_EP_APP_PASSWORD', variable: 'BSKY_EP_APP_PASSWORD') + string(credentialsId: 'BSKY_EP_HANDLE', variable: 'BSKY_EP_HANDLE'), + string(credentialsId: 'BSKY_EP_USERNAME', variable: 'BSKY_EP_USERNAME'), + string(credentialsId: 'BSKY_EP_APP_PASSWORD', variable: 'BSKY_EP_APP_PASSWORD') ]) { script { def feeds = [ @@ -85,9 +85,10 @@ H 22,10,16 * * * int batchSize = env.MAX_PARALLEL_FEEDS as int int jitterMax = env.JITTER_MAX_SECONDS as int - def batches = feeds.collate(batchSize) + int maxPosts = env.MAX_POSTS_PER_FEED as int + def batches = feeds.collate(batchSize) - echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}" + echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}, max posts/feed: ${maxPosts}" int batchNum = 0 for (def batch : batches) { @@ -98,7 +99,7 @@ H 22,10,16 * * * batch.each { feed -> def feedName = feed.name - def feedUrl = feed.url + def feedUrl = feed.url parallelTasks[feedName] = { stage("Feed: ${feedName}") { @@ -115,8 +116,11 @@ H 22,10,16 * * * "${feedUrl}" \\ "\$BSKY_EP_HANDLE" \\ "\$BSKY_EP_USERNAME" \\ - "\$BSKY_EP_APP_PASSWORD" \ - --service "https://eurosky.social" + "\$BSKY_EP_APP_PASSWORD" \\ + --service "https://eurosky.social" \\ + --state-path "state_${feedName}.json" \\ + --cooldown-path "cooldown_${feedName}.json" \\ + --max-posts "${maxPosts}" """ } } @@ -124,7 +128,14 @@ H 22,10,16 * * * } parallel parallelTasks + echo "Finished batch ${batchNum}/${batches.size()}" + + // Stagger batches to reduce cumulative API pressure + if (batchNum < batches.size()) { + echo "âŗ Sleeping 15s between batches..." + sleep(time: 15, unit: 'SECONDS') + } } } } @@ -134,6 +145,7 @@ H 22,10,16 * * * post { always { + // Archive all per-feed state and cooldown files + any logs archiveArtifacts artifacts: '*.json, **/*.log', allowEmptyArchive: true } unstable { diff --git a/rss2bsky.py b/rss2bsky.py index 7f61439..cb563fb 100644 --- a/rss2bsky.py +++ b/rss2bsky.py @@ -65,8 +65,8 @@ class RetryConfig: @dataclass(frozen=True) class CooldownConfig: - default_post_cooldown_seconds: int = 3600 - default_thumb_cooldown_seconds: int = 1800 + default_post_cooldown_seconds: int = 120 + default_thumb_cooldown_seconds: int = 60 @dataclass(frozen=True) @@ -1147,16 +1147,18 @@ def login_with_backoff( return False return False + def run_once( rss_feed: str, bsky_handle: str, bsky_username: str, bsky_password: str, service_url: str, - post_langs: List[str], # ← changed from post_lang: str + post_langs: List[str], state_path: str, cooldown_path: str, - cfg: AppConfig + cfg: AppConfig, + max_posts: int = 5 # ← NEW PARAMETER ) -> RunResult: if not PIL_AVAILABLE: logging.warning("🟡 Pillow is not installed. External card thumbnail compression is disabled.") @@ -1210,6 +1212,10 @@ def run_once( logging.info(f"đŸ“Ŧ {len(entries_to_post)} entries remain after duplicate filtering.") + # ← NEW: log the effective cap before starting the loop + if len(entries_to_post) > max_posts: + logging.info(f"đŸ”ĸ max-posts cap is {max_posts}: will publish at most {max_posts} of {len(entries_to_post)} entries this run.") + if not entries_to_post: logging.info("ℹī¸ Execution finished: no new entries to publish.") return RunResult(published_count=0) @@ -1220,6 +1226,12 @@ def run_once( published = 0 for candidate in entries_to_post: + + # ← NEW: hard cap check at the top of every iteration + if published >= max_posts: + logging.info(f"đŸ”ĸ === MAX POSTS REACHED === Stopping after {published} posts (limit: {max_posts}).") + break + if is_global_post_cooldown_active(cooldown_path): reset_str = format_cooldown_until(get_global_post_cooldown_until(cooldown_path)) logging.error(f"🛑 === BSKY POST STOPPED: GLOBAL COOLDOWN === Skipping remaining entries until {reset_str}") @@ -1248,7 +1260,7 @@ def run_once( client=client, text_variants=text_variants, embed=embed, - post_langs=post_langs, # ← changed + post_langs=post_langs, cooldown_path=cooldown_path, cfg=cfg ) @@ -1318,6 +1330,7 @@ def main(): ) parser.add_argument("--state-path", default=DEFAULT_STATE_PATH, help="Path to local JSON state file") parser.add_argument("--cooldown-path", default=DEFAULT_COOLDOWN_STATE_PATH, help="Path to shared cooldown JSON state file") + parser.add_argument('--max-posts', type=int, default=5, help='Max new posts to publish per run') args = parser.parse_args() # Parse comma-separated langs: "ca,es" → ["ca", "es"] @@ -1335,10 +1348,11 @@ def main(): bsky_username=args.bsky_username, bsky_password=args.bsky_app_password, service_url=args.service, - post_langs=post_langs, # ← changed + post_langs=args.lang.split(","), state_path=args.state_path, cooldown_path=args.cooldown_path, - cfg=cfg + cfg=AppConfig(), + max_posts=args.max_posts )