Skip to content

Commit 0b76bd5

Browse files
KamiKami
committed
Store log files for each screen session inside launchdev script.
This way we can troubleshoot various "fail to start" issues on CI. Without this change, we have no visibility if service fails to start before the actual service starts writting into it's own log file. Sadly we can't rely on new version of screen which supports -Logfile path.log flag so we need to use screen config per screen window so we can use different log files for different processes.
1 parent e446a46 commit 0b76bd5

14 files changed

+101
-19
lines changed

‎scripts/travis/prepare-integration.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,17 @@ source ./virtualenv/bin/activate
1515
python ./st2client/setup.py develop
1616
st2 --version
1717

18+
# Clean up old screen log files
19+
rm -f logs/screen-*.log
20+
1821
# start dev environment in screens
1922
./tools/launchdev.sh start -x
2023

24+
# Give processes some time to start and check logs to see if all the services
25+
# started or if there was any error / failure
26+
sleep 10
27+
cat logs/screen-*.log
28+
2129
# This script runs as root on Travis which means other processes which don't run
2230
# as root can't write to logs/ directory and tests fail
2331
chmod 777 logs/

‎tools/launchdev.sh‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -245,15 +245,18 @@ function st2start(){
245245
screen -ls | grep mistral | cut -d. -f1 | awk '{print $1}' | xargs kill
246246
fi
247247

248+
# NOTE: We can't rely on latest version of screen with "-Logfile path"
249+
# option so we need to use screen config file per screen window
250+
248251
# Run the st2 API server
249252
echo 'Starting screen session st2-api...'
250253
if [ "${use_gunicorn}" = true ]; then
251254
echo ' using gunicorn to run st2-api...'
252255
export ST2_CONFIG_PATH=${ST2_CONF}
253-
screen -d -m -S st2-api ${VIRTUALENV}/bin/gunicorn \
256+
screen -L -c tools/screen-configs/st2api.conf -d -m -S st2-api ${VIRTUALENV}/bin/gunicorn \
254257
st2api.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9101" --workers 1
255258
else
256-
screen -d -m -S st2-api ${VIRTUALENV}/bin/python \
259+
screen -L -c tools/screen-configs/st2api.conf -d -m -S st2-api ${VIRTUALENV}/bin/python \
257260
./st2api/bin/st2api \
258261
--config-file $ST2_CONF
259262
fi
@@ -262,10 +265,10 @@ function st2start(){
262265
if [ "${use_gunicorn}" = true ]; then
263266
echo ' using gunicorn to run st2-stream'
264267
export ST2_CONFIG_PATH=${ST2_CONF}
265-
screen -d -m -S st2-stream ${VIRTUALENV}/bin/gunicorn \
268+
screen -L -c tools/screen-configs/st2stream.conf -d -m -S st2-stream ${VIRTUALENV}/bin/gunicorn \
266269
st2stream.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9102" --workers 1
267270
else
268-
screen -d -m -S st2-stream ${VIRTUALENV}/bin/python \
271+
screen -L -c tools/screen-configs/st2stream.conf -d -m -S st2-stream ${VIRTUALENV}/bin/python \
269272
./st2stream/bin/st2stream \
270273
--config-file $ST2_CONF
271274
fi
@@ -278,7 +281,7 @@ function st2start(){
278281
WORKFLOW_ENGINE_NAME=st2-workflow-$i
279282
WORKFLOW_ENGINE_SCREENS+=($WORKFLOW_ENGINE_NAME)
280283
echo ' starting '$WORKFLOW_ENGINE_NAME'...'
281-
screen -d -m -S $WORKFLOW_ENGINE_NAME ${VIRTUALENV}/bin/python \
284+
screen -L -c tools/screen-configs/st2workflowengine.conf -d -m -S $WORKFLOW_ENGINE_NAME ${VIRTUALENV}/bin/python \
282285
./st2actions/bin/st2workflowengine \
283286
--config-file $ST2_CONF
284287
done
@@ -291,14 +294,14 @@ function st2start(){
291294
RUNNER_NAME=st2-actionrunner-$i
292295
RUNNER_SCREENS+=($RUNNER_NAME)
293296
echo ' starting '$RUNNER_NAME'...'
294-
screen -d -m -S $RUNNER_NAME ${VIRTUALENV}/bin/python \
297+
screen -L -c tools/screen-configs/st2actionrunner.conf -d -m -S $RUNNER_NAME ${VIRTUALENV}/bin/python \
295298
./st2actions/bin/st2actionrunner \
296299
--config-file $ST2_CONF
297300
done
298301

299302
# Run the garbage collector service
300303
echo 'Starting screen session st2-garbagecollector'
301-
screen -d -m -S st2-garbagecollector ${VIRTUALENV}/bin/python \
304+
screen -L -c tools/screen-configs/st2garbagecollector.conf -d -m -S st2-garbagecollector ${VIRTUALENV}/bin/python \
302305
./st2reactor/bin/st2garbagecollector \
303306
--config-file $ST2_CONF
304307

@@ -310,38 +313,38 @@ function st2start(){
310313
SCHEDULER_NAME=st2-scheduler-$i
311314
SCHEDULER_SCREENS+=($SCHEDULER_NAME)
312315
echo ' starting '$SCHEDULER_NAME'...'
313-
screen -d -m -S $SCHEDULER_NAME ${VIRTUALENV}/bin/python \
316+
screen -L -c tools/screen-configs/st2scheduler.conf -d -m -S $SCHEDULER_NAME ${VIRTUALENV}/bin/python \
314317
./st2actions/bin/st2scheduler \
315318
--config-file $ST2_CONF
316319
done
317320

318321
# Run the sensor container server
319322
echo 'Starting screen session st2-sensorcontainer'
320-
screen -d -m -S st2-sensorcontainer ${VIRTUALENV}/bin/python \
323+
screen -L -c tools/screen-configs/st2sensorcontainer.conf -d -m -S st2-sensorcontainer ${VIRTUALENV}/bin/python \
321324
./st2reactor/bin/st2sensorcontainer \
322325
--config-file $ST2_CONF
323326

324327
# Run the rules engine server
325328
echo 'Starting screen session st2-rulesengine...'
326-
screen -d -m -S st2-rulesengine ${VIRTUALENV}/bin/python \
329+
screen -L -c tools/screen-configs/st2rulesengine.conf -d -m -S st2-rulesengine ${VIRTUALENV}/bin/python \
327330
./st2reactor/bin/st2rulesengine \
328331
--config-file $ST2_CONF
329332

330333
# Run the timer engine server
331334
echo 'Starting screen session st2-timersengine...'
332-
screen -d -m -S st2-timersengine ${VIRTUALENV}/bin/python \
335+
screen -L -c tools/screen-configs/st2timersengine.conf -d -m -S st2-timersengine ${VIRTUALENV}/bin/python \
333336
./st2reactor/bin/st2timersengine \
334337
--config-file $ST2_CONF
335338

336339
# Run the results tracker
337340
echo 'Starting screen session st2-resultstracker...'
338-
screen -d -m -S st2-resultstracker ${VIRTUALENV}/bin/python \
341+
screen -L -c tools/screen-configs/st2resultstracker.conf -d -m -S st2-resultstracker ${VIRTUALENV}/bin/python \
339342
./st2actions/bin/st2resultstracker \
340343
--config-file $ST2_CONF
341344

342345
# Run the actions notifier
343346
echo 'Starting screen session st2-notifier...'
344-
screen -d -m -S st2-notifier ${VIRTUALENV}/bin/python \
347+
screen -L -c tools/screen-configs/st2notifier.conf -d -m -S st2-notifier ${VIRTUALENV}/bin/python \
345348
./st2actions/bin/st2notifier \
346349
--config-file $ST2_CONF
347350

@@ -350,10 +353,10 @@ function st2start(){
350353
if [ "${use_gunicorn}" = true ]; then
351354
echo ' using gunicorn to run st2-auth...'
352355
export ST2_CONFIG_PATH=${ST2_CONF}
353-
screen -d -m -S st2-auth ${VIRTUALENV}/bin/gunicorn \
356+
screen -L -c tools/screen-configs/st2auth.conf -d -m -S st2-auth ${VIRTUALENV}/bin/gunicorn \
354357
st2auth.wsgi:application -k eventlet -b "$BINDING_ADDRESS:9100" --workers 1
355358
else
356-
screen -d -m -S st2-auth ${VIRTUALENV}/bin/python \
359+
screen -L -c tools/screen-configs/st2auth.conf -d -m -S st2-auth ${VIRTUALENV}/bin/python \
357360
./st2auth/bin/st2auth \
358361
--config-file $ST2_CONF
359362
fi
@@ -364,26 +367,25 @@ function st2start(){
364367
sudo mkdir -p $EXPORTS_DIR
365368
sudo chown -R ${CURRENT_USER}:${CURRENT_USER_GROUP} $EXPORTS_DIR
366369
echo 'Starting screen session st2-exporter...'
367-
screen -d -m -S st2-exporter ${VIRTUALENV}/bin/python \
370+
screen -L -d -m -S st2-exporter ${VIRTUALENV}/bin/python \
368371
./st2exporter/bin/st2exporter \
369372
--config-file $ST2_CONF
370373
fi
371374

372375
if [ "${include_mistral}" = true ]; then
373-
374376
LOGDIR=${ST2_REPO}/logs
375377

376378
# Run mistral-server
377379
echo 'Starting screen session mistral-server...'
378-
screen -d -m -S mistral-server ${MISTRAL_REPO}/.venv/bin/python \
380+
screen -L -Logfile logs/screen-mistral-server.log -d -m -S mistral-server ${MISTRAL_REPO}/.venv/bin/python \
379381
${MISTRAL_REPO}/.venv/bin/mistral-server \
380382
--server engine,executor \
381383
--config-file $MISTRAL_CONF \
382384
--log-file "$LOGDIR/mistral-server.log"
383385

384386
# Run mistral-api
385387
echo 'Starting screen session mistral-api...'
386-
screen -d -m -S mistral-api ${MISTRAL_REPO}/.venv/bin/python \
388+
screen -L -Logfile logs/screen-mistral-server.log -d -m -S mistral-api ${MISTRAL_REPO}/.venv/bin/python \
387389
${MISTRAL_REPO}/.venv/bin/mistral-server \
388390
--server api \
389391
--config-file $MISTRAL_CONF \

‎tools/screen-configs/st2actionrunner.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2actionrunner.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2api.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2api.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2auth.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2auth.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2garbagecollector.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2garbagecollector.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2notifier.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2notifier.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2resultstracker.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2resultstracker.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2rulesengine.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2rulesengine.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

‎tools/screen-configs/st2scheduler.conf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
logfile logs/screen-st2scheduler.log
2+
logfile flush 1
3+
log on
4+
logtstamp after 1
5+
logtstamp string \"[ %t: %Y-%m-%d %c:%s ]\012\"
6+
logtstamp on

0 commit comments

Comments
 (0)