149 lines
7.1 KiB
Plaintext
149 lines
7.1 KiB
Plaintext
pipeline {
|
|
agent any
|
|
|
|
options {
|
|
timeout(time: 30, unit: 'MINUTES')
|
|
timestamps()
|
|
buildDiscarder(logRotator(numToKeepStr: '20'))
|
|
disableConcurrentBuilds()
|
|
}
|
|
|
|
triggers {
|
|
cron('''
|
|
H 22,10,16 * * *
|
|
0 4 * * *
|
|
''')
|
|
}
|
|
|
|
|
|
environment {
|
|
VENV_DIR = 'venv'
|
|
MAX_PARALLEL_FEEDS = '4'
|
|
JITTER_MAX_SECONDS = '12'
|
|
PYTHONUNBUFFERED = '1'
|
|
PIP_CACHE_DIR = "${WORKSPACE}/.pip-cache"
|
|
}
|
|
|
|
stages {
|
|
stage('Checkout') {
|
|
steps {
|
|
checkout scm
|
|
}
|
|
}
|
|
|
|
stage('Setup Python') {
|
|
steps {
|
|
sh '''
|
|
set -euxo pipefail
|
|
python3 -m venv "${VENV_DIR}"
|
|
. "${VENV_DIR}/bin/activate"
|
|
python -m pip install --upgrade pip wheel setuptools
|
|
pip install --cache-dir "${PIP_CACHE_DIR}" \
|
|
atproto fastfeedparser beautifulsoup4 httpx arrow charset-normalizer Pillow
|
|
python --version
|
|
pip --version
|
|
'''
|
|
}
|
|
}
|
|
|
|
stage('Process All RSS Feeds (Batched Parallel)') {
|
|
steps {
|
|
withCredentials([
|
|
string(credentialsId: 'BSKY_EP_HANDLE', variable: 'BSKY_EP_HANDLE'),
|
|
string(credentialsId: 'BSKY_EP_USERNAME', variable: 'BSKY_EP_USERNAME'),
|
|
string(credentialsId: 'BSKY_EP_APP_PASSWORD', variable: 'BSKY_EP_APP_PASSWORD')
|
|
]) {
|
|
script {
|
|
def feeds = [
|
|
[name: 'badalona', url: 'https://www.elperiodico.cat/ca/rss/badalona/rss.xml'],
|
|
[name: 'barcelona', url: 'https://www.elperiodico.cat/ca/rss/barcelona/rss.xml'],
|
|
[name: 'ciencia', url: 'https://www.elperiodico.cat/ca/rss/ciencia/rss.xml'],
|
|
[name: 'cornella', url: 'https://www.elperiodico.cat/ca/rss/cornella/rss.xml'],
|
|
[name: 'economia', url: 'https://www.elperiodico.cat/ca/rss/economia/rss.xml'],
|
|
[name: 'educacio', url: 'https://www.elperiodico.cat/ca/rss/educacio/rss.xml'],
|
|
[name: 'esports', url: 'https://www.elperiodico.cat/ca/rss/esports/rss.xml'],
|
|
[name: 'extra', url: 'https://www.elperiodico.cat/ca/rss/extra/rss.xml'],
|
|
[name: 'gent', url: 'https://www.elperiodico.cat/ca/rss/gent/rss.xml'],
|
|
[name: 'hospitalet', url: 'https://www.elperiodico.cat/ca/rss/hospitalet/rss.xml'],
|
|
[name: 'internacional', url: 'https://www.elperiodico.cat/ca/rss/internacional/rss.xml'],
|
|
[name: 'medi-ambient', url: 'https://www.elperiodico.cat/ca/rss/medi-ambient/rss.xml'],
|
|
[name: 'motor', url: 'https://www.elperiodico.cat/ca/rss/motor/rss.xml'],
|
|
[name: 'oci-i-cultura', url: 'https://www.elperiodico.cat/ca/rss/oci-i-cultura/rss.xml'],
|
|
[name: 'opinio', url: 'https://www.elperiodico.cat/ca/rss/opinio/rss.xml'],
|
|
[name: 'politica', url: 'https://www.elperiodico.cat/ca/rss/politica/rss.xml'],
|
|
[name: 'portada', url: 'https://www.elperiodico.cat/ca/rss/portada/rss.xml'],
|
|
[name: 'que-fer', url: 'https://www.elperiodico.cat/ca/rss/que-fer/rss.xml'],
|
|
[name: 'sabadell', url: 'https://www.elperiodico.cat/ca/rss/sabadell/rss.xml'],
|
|
[name: 'sanitat', url: 'https://www.elperiodico.cat/ca/rss/sanitat/rss.xml'],
|
|
[name: 'santa-coloma', url: 'https://www.elperiodico.cat/ca/rss/santa-coloma/rss.xml'],
|
|
[name: 'societat', url: 'https://www.elperiodico.cat/ca/rss/societat/rss.xml'],
|
|
[name: 'tecnologia', url: 'https://www.elperiodico.cat/ca/rss/tecnologia/rss.xml'],
|
|
[name: 'tele', url: 'https://www.elperiodico.cat/ca/rss/tele/rss.xml'],
|
|
[name: 'temps', url: 'https://www.elperiodico.cat/ca/rss/temps/rss.xml'],
|
|
[name: 'terrassa', url: 'https://www.elperiodico.cat/ca/rss/terrassa/rss.xml']
|
|
]
|
|
|
|
int batchSize = env.MAX_PARALLEL_FEEDS as int
|
|
int jitterMax = env.JITTER_MAX_SECONDS as int
|
|
def batches = feeds.collate(batchSize)
|
|
|
|
echo "Total feeds: ${feeds.size()}, batch size: ${batchSize}, batches: ${batches.size()}"
|
|
|
|
int batchNum = 0
|
|
for (def batch : batches) {
|
|
batchNum++
|
|
echo "Starting batch ${batchNum}/${batches.size()} with ${batch.size()} feeds"
|
|
|
|
def parallelTasks = [:]
|
|
|
|
batch.each { feed ->
|
|
def feedName = feed.name
|
|
def feedUrl = feed.url
|
|
|
|
parallelTasks[feedName] = {
|
|
stage("Feed: ${feedName}") {
|
|
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
|
|
sh """
|
|
set -euxo pipefail
|
|
. "${VENV_DIR}/bin/activate"
|
|
|
|
JITTER=\$((RANDOM % ${jitterMax}))
|
|
echo "[${feedName}] sleeping \$JITTER s jitter"
|
|
sleep \$JITTER
|
|
|
|
python3 rss2bsky.py \\
|
|
"${feedUrl}" \\
|
|
"\$BSKY_EP_HANDLE" \\
|
|
"\$BSKY_EP_USERNAME" \\
|
|
"\$BSKY_EP_APP_PASSWORD" \
|
|
--service "https://eurosky.social"
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
parallel parallelTasks
|
|
echo "Finished batch ${batchNum}/${batches.size()}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
always {
|
|
archiveArtifacts artifacts: '*.json, **/*.log', allowEmptyArchive: true
|
|
}
|
|
unstable {
|
|
echo 'Build unstable: one or more feeds failed, but pipeline completed.'
|
|
}
|
|
failure {
|
|
echo 'Build failed.'
|
|
}
|
|
success {
|
|
echo 'Build succeeded.'
|
|
}
|
|
}
|
|
} |