Agile611/post2bsky
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(jenkins): isolate per-feed state/cooldown, cap posts, stagger batches

Browse Source 
- Pass --state-path and --cooldown-path per feed so a rate limit on one
  feed no longer triggers a global cooldown that blocks all other feeds
- Add --max-posts 5 cap to prevent burst posting (e.g. 22 posts in one
  run) that was causing HTTP 429 errors on eurosky.social
- Add 15s sleep between batches to reduce cumulative API pressure
- Increase MAX_PARALLEL_FEEDS from 4 to 6 now that cooldowns are isolated
- Add MAX_POSTS_PER_FEED env var for central control of the post cap
- Fix missing line continuation backslash on --service argument

Fixes: batch 2-7 feeds being skipped due to shared cooldown state
This commit is contained in:
Guillem Hernandez Sola
2026-05-13 07:05:46 +02:00
parent fb30fc5e3a
commit 7b73a40ed7
2 changed files with 49 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
jenkins/ePrss
View File

@@ -8,20 +8,20 @@ pipeline {
disableConcurrentBuilds()
}
triggers {
cron('''
triggers {
cron('''
H 22,10,16 * * *
0 6 * * *
''')
}
}
environment {
VENV_DIR = 'venv'
MAX_PARALLEL_FEEDS = '4'
VENV_DIR = 'venv'
MAX_PARALLEL_FEEDS = '6'
JITTER_MAX_SECONDS = '12'
PYTHONUNBUFFERED = '1'
PIP_CACHE_DIR = "${WORKSPACE}/.pip-cache"
MAX_POSTS_PER_FEED = '5'
PYTHONUNBUFFERED = '1'
PIP_CACHE_DIR = "${WORKSPACE}/.pip-cache"
}
stages {
@@ -49,9 +49,9 @@ H 22,10,16 * * *
stage('Process All RSS Feeds (Batched Parallel)') {
steps {
withCredentials([
string(credentialsId: 'BSKY_EP_HANDLE', variable: 'BSKY_EP_HANDLE'),
string(credentialsId: 'BSKY_EP_USERNAME', variable: 'BSKY_EP_USERNAME'),
string(credentialsId: 'BSKY_EP_APP_PASSWORD', variable: 'BSKY_EP_APP_PASSWORD')
string(credentialsId: 'BSKY_EP_HANDLE', variable: 'BSKY_EP_HANDLE'),
string(credentialsId: 'BSKY_EP_USERNAME', variable: 'BSKY_EP_USERNAME'),
string(credentialsId: 'BSKY_EP_APP_PASSWORD', variable: 'BSKY_EP_APP_PASSWORD')
]) {
script {
def feeds = [
@@ -85,9 +85,10 @@ H 22,10,16 * * *
int batchSize = env.MAX_PARALLEL_FEEDS as int
int jitterMax = env.JITTER_MAX_SECONDS as int
def batches = feeds.collate(batchSize)
int maxPosts = env.MAX_POSTS_PER_FEED as int
def batches = feeds.collate(batchSize)
echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}"
echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}, max posts/feed: ${maxPosts}"
int batchNum = 0
for (def batch : batches) {
@@ -98,7 +99,7 @@ H 22,10,16 * * *
batch.each { feed ->
def feedName = feed.name
def feedUrl = feed.url
def feedUrl = feed.url
parallelTasks[feedName] = {
stage("Feed: ${feedName}") {
@@ -115,8 +116,11 @@ H 22,10,16 * * *
"${feedUrl}" \\
"\$BSKY_EP_HANDLE" \\
"\$BSKY_EP_USERNAME" \\
"\$BSKY_EP_APP_PASSWORD" \
--service "https://eurosky.social"
"\$BSKY_EP_APP_PASSWORD" \\
--service "https://eurosky.social" \\
--state-path "state_${feedName}.json" \\
--cooldown-path "cooldown_${feedName}.json" \\
--max-posts "${maxPosts}"
"""
}
}
@@ -124,7 +128,14 @@ H 22,10,16 * * *
}
parallel parallelTasks
echo "Finished batch ${batchNum}/${batches.size()}"
// Stagger batches to reduce cumulative API pressure
if (batchNum < batches.size()) {
echo "⏳ Sleeping 15s between batches..."
sleep(time: 15, unit: 'SECONDS')
}
}
}
}
@@ -134,6 +145,7 @@ H 22,10,16 * * *
post {
always {
// Archive all per-feed state and cooldown files + any logs
archiveArtifacts artifacts: '*.json, **/*.log', allowEmptyArchive: true
}
unstable {