Agile611/post2bsky
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(jenkins): isolate per-feed state/cooldown, cap posts, stagger batches

Browse Source 
- Pass --state-path and --cooldown-path per feed so a rate limit on one
  feed no longer triggers a global cooldown that blocks all other feeds
- Add --max-posts 5 cap to prevent burst posting (e.g. 22 posts in one
  run) that was causing HTTP 429 errors on eurosky.social
- Add 15s sleep between batches to reduce cumulative API pressure
- Increase MAX_PARALLEL_FEEDS from 4 to 6 now that cooldowns are isolated
- Add MAX_POSTS_PER_FEED env var for central control of the post cap
- Fix missing line continuation backslash on --service argument

Fixes: batch 2-7 feeds being skipped due to shared cooldown state
This commit is contained in:
Guillem Hernandez Sola
2026-05-13 07:05:46 +02:00
parent fb30fc5e3a
commit 7b73a40ed7
2 changed files with 49 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
jenkins/ePrss
View File

@@ -8,18 +8,18 @@ pipeline {
disableConcurrentBuilds()
}
triggers {
triggers {
cron('''
H 22,10,16 * * *
0 6 * * *
''')
}
}
environment {
VENV_DIR = 'venv'
MAX_PARALLEL_FEEDS = '4'
MAX_PARALLEL_FEEDS = '6'
JITTER_MAX_SECONDS = '12'
MAX_POSTS_PER_FEED = '5'
PYTHONUNBUFFERED = '1'
PIP_CACHE_DIR = "${WORKSPACE}/.pip-cache"
}
@@ -85,9 +85,10 @@ H 22,10,16 * * *
int batchSize = env.MAX_PARALLEL_FEEDS as int
int jitterMax = env.JITTER_MAX_SECONDS as int
int maxPosts = env.MAX_POSTS_PER_FEED as int
def batches = feeds.collate(batchSize)
echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}"
echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}, max posts/feed: ${maxPosts}"
int batchNum = 0
for (def batch : batches) {
@@ -115,8 +116,11 @@ H 22,10,16 * * *
"${feedUrl}" \\
"\$BSKY_EP_HANDLE" \\
"\$BSKY_EP_USERNAME" \\
"\$BSKY_EP_APP_PASSWORD" \
--service "https://eurosky.social"
"\$BSKY_EP_APP_PASSWORD" \\
--service "https://eurosky.social" \\
--state-path "state_${feedName}.json" \\
--cooldown-path "cooldown_${feedName}.json" \\
--max-posts "${maxPosts}"
"""
}
}
@@ -124,7 +128,14 @@ H 22,10,16 * * *
}
parallel parallelTasks
echo "Finished batch ${batchNum}/${batches.size()}"
// Stagger batches to reduce cumulative API pressure
if (batchNum < batches.size()) {
echo "⏳ Sleeping 15s between batches..."
sleep(time: 15, unit: 'SECONDS')
}
}
}
}
@@ -134,6 +145,7 @@ H 22,10,16 * * *
post {
always {
// Archive all per-feed state and cooldown files + any logs
archiveArtifacts artifacts: '*.json, **/*.log', allowEmptyArchive: true
}
unstable {

28
rss2bsky.py
View File

@@ -65,8 +65,8 @@ class RetryConfig:
@dataclass(frozen=True)
class CooldownConfig:
default_post_cooldown_seconds: int = 3600
default_thumb_cooldown_seconds: int = 1800
default_post_cooldown_seconds: int = 120
default_thumb_cooldown_seconds: int = 60
@dataclass(frozen=True)
@@ -1147,16 +1147,18 @@ def login_with_backoff(
return False
return False
def run_once(
rss_feed: str,
bsky_handle: str,
bsky_username: str,
bsky_password: str,
service_url: str,
post_langs: List[str], # ← changed from post_lang: str
post_langs: List[str],
state_path: str,
cooldown_path: str,
cfg: AppConfig
cfg: AppConfig,
max_posts: int = 5 # ← NEW PARAMETER
) -> RunResult:
if not PIL_AVAILABLE:
logging.warning("🟡 Pillow is not installed. External card thumbnail compression is disabled.")
@@ -1210,6 +1212,10 @@ def run_once(
logging.info(f"📬 {len(entries_to_post)} entries remain after duplicate filtering.")
# ← NEW: log the effective cap before starting the loop
if len(entries_to_post) > max_posts:
logging.info(f"🔢 max-posts cap is {max_posts}: will publish at most {max_posts} of {len(entries_to_post)} entries this run.")
if not entries_to_post:
logging.info("ℹ️ Execution finished: no new entries to publish.")
return RunResult(published_count=0)
@@ -1220,6 +1226,12 @@ def run_once(
published = 0
for candidate in entries_to_post:
# ← NEW: hard cap check at the top of every iteration
if published >= max_posts:
logging.info(f"🔢 === MAX POSTS REACHED === Stopping after {published} posts (limit: {max_posts}).")
break
if is_global_post_cooldown_active(cooldown_path):
reset_str = format_cooldown_until(get_global_post_cooldown_until(cooldown_path))
logging.error(f"🛑 === BSKY POST STOPPED: GLOBAL COOLDOWN === Skipping remaining entries until {reset_str}")
@@ -1248,7 +1260,7 @@ def run_once(
client=client,
text_variants=text_variants,
embed=embed,
post_langs=post_langs, # ← changed
post_langs=post_langs,
cooldown_path=cooldown_path,
cfg=cfg
)
@@ -1318,6 +1330,7 @@ def main():
)
parser.add_argument("--state-path", default=DEFAULT_STATE_PATH, help="Path to local JSON state file")
parser.add_argument("--cooldown-path", default=DEFAULT_COOLDOWN_STATE_PATH, help="Path to shared cooldown JSON state file")
parser.add_argument('--max-posts', type=int, default=5, help='Max new posts to publish per run')
args = parser.parse_args()
# Parse comma-separated langs: "ca,es" → ["ca", "es"]
@@ -1335,10 +1348,11 @@ def main():
bsky_username=args.bsky_username,
bsky_password=args.bsky_app_password,
service_url=args.service,
post_langs=post_langs, # ← changed
post_langs=args.lang.split(","),
state_path=args.state_path,
cooldown_path=args.cooldown_path,
cfg=cfg
cfg=AppConfig(),
max_posts=args.max_posts
)