FAELIX
/
incident_handler
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

massive restructure

Browse Source
master
Marek Isalski 7 years ago
parent 82126a1f51
commit 27afe2229f
2 changed files with 162 additions and 136 deletions
Show Stats Download Patch File Download Diff File

283
fih.py
Unescape Escape View File

@ -5,7 +5,16 @@ import incident
import datetime import datetime
import subprocess import subprocess
import re import re
import random
import mattermostdriver
random.seed( os.urandom( 8 ) )
class Mattermost( object ):
def __init__( self, *args, **kwargs ):
self.mm = mattermostdriver.Driver( kwargs )
self.mm.login()
def strip_ansi( text, escape = re.compile( r'\x1B\[[0-?]*[ -/]*[@-~]' ) ): def strip_ansi( text, escape = re.compile( r'\x1B\[[0-?]*[ -/]*[@-~]' ) ):
return escape.sub( '', text ) return escape.sub( '', text )
@ -17,49 +26,123 @@ def strip_progress( text, escape = re.compile( r'\r[^\n]*\n' ) ):
app = Flask( __name__ ) app = Flask( __name__ )
app.config.from_envvar( 'FIH_SETTINGS' ) app.config.from_envvar( 'FIH_SETTINGS' )
app.mm = incident.Mattermost( url = app.config[ 'MATTERMOST_URL' ], app.mm = Mattermost( url = app.config[ 'MATTERMOST_URL' ],
port = app.config[ 'MATTERMOST_PORT' ], port = app.config[ 'MATTERMOST_PORT' ],
login_id = app.config[ 'MATTERMOST_USER_EMAIL' ], login_id = app.config[ 'MATTERMOST_USER_EMAIL' ],
password = app.config[ 'MATTERMOST_USER_PASSWORD' ], password = app.config[ 'MATTERMOST_USER_PASSWORD' ],
).mm ).mm
@app.route( '/mattermost/incident', methods = [ 'POST' ] )
def mattermost_incident():
# channel_name: testing
# command: /incident
# channel_id: o3ob1utim3nummoswjyt4174mo
# user_name: marek
# test_domain: faelix
# text: test
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
# token: XXXXXXXXXXXXXXXXXXXXXXXXXX
if request.form[ 'token' ] in app.config[ 'MATTERMOST_COMMAND_TOKENS' ]: def generate_raw_incident_number():
incident_number = incident.generate_incident_number() return datetime.datetime.now().strftime( "%Y%m%d" ) + ( "%4d" % int( random.random() * 10000 ) )
fi_number = "FI#" + incident_number
channel_name = "incident" + incident_number def raw_incident_number_to_channel_name( incident_number ):
purpose = request.form[ 'text' ] return app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] + incident_number
channel = app.mm.channels.create_channel( options = { "team_id": request.form[ 'team_id' ], def raw_incident_number_to_public_incident_number( incident_number ):
"name": channel_name, return app.config[ "PUBLIC_INCIDENT_PREFIX" ] + incident_number
"display_name": fi_number,
"purpose": purpose, def raw_incident_number_to_url( incident_number ):
"header": "", return app.config[ "INCIDENT_URL_SCHEME" ] % incident_number
"type": "O",
} ) def is_incident_channel_name( channel_name ):
return channel_name.startswith( app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] )
# add user to the channel we just created
app.mm.client.make_request( 'post', '/channels/' + channel[ 'id' ] + '/members', options = { 'user_id': request.form[ 'user_id' ],
} )
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
'message': "Incident discussion channel for [%s](https://status.faelix.net/incident/%s) created by @%s." % ( fi_number, incident_number, request.form[ 'user_name' ] ),
} )
now = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S" ) def mattermost_incident_command():
cmd = request.form[ 'text' ].strip().split()[ 0 ].upper()
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ], app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': """\ 'message': "### Executing `" + cmd + "`...",
} )
if cmd == "PUBLISH":
cmdline = app.config[ "MERCURIAL_BIN" ] + " update"
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
cmdline = app.config[ "MERCURIAL_BIN" ] + " pull " + pull_from
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
cmdline = app.config[ "MERCURIAL_BIN" ] + " merge"
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
cmdline = app.config[ "MERCURIAL_BIN" ] + " commit -m 'merge from " + pull_from + "'"
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
for deploy_to in app.config[ "GROW_DEPLOY_TO" ]:
cmdline = app.config[ "GROW_BIN" ] + " deploy -f " + deploy_to
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
elif cmd == "START":
pass
elif cmd == "ETR":
pass
elif cmd == "RESOLVED":
pass
elif cmd == "CLOSED":
pass
elif cmd == "TIMELINE":
pass
elif cmd == "INCIDENT":
pass
elif cmd == "DEGRADED":
pass
elif cmd == "MAINTENANCE":
pass
elif cmd == "NOTICE":
pass
elif cmd == "OK":
pass
else:
return "WTF command?"
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': "### Finished `" + cmd + "`!",
} )
return "OK"
def mattermost_incident_start():
raw_incident_number = incident.generate_raw_incident_number()
fi_number = raw_incident_number_to_public_incident_number( raw_incident_number )
channel_name = raw_incident_number_to_channel_name( raw_incident_number )
purpose = request.form[ 'text' ]
channel = app.mm.channels.create_channel( options = { "team_id": request.form[ 'team_id' ],
"name": channel_name,
"display_name": fi_number,
"purpose": purpose,
"header": "",
"type": "O",
} )
# add user to the channel we just created
app.mm.client.make_request( 'post', '/channels/' + channel[ 'id' ] + '/members', options = { 'user_id': request.form[ 'user_id' ],
} )
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
'message': "Incident discussion channel for [%s](%s) created by @%s." % ( fi_number, raw_incident_number_to_url( incident_number ), request.form[ 'user_name' ] ),
} )
now = datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" )
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
'message': """\
# Incident Response Communication Procedure # Incident Response Communication Procedure
The contents of this channel are strictly confidential. The contents of this channel are strictly confidential.
@ -69,39 +152,57 @@ The contents of this channel are strictly confidential.
1. adjust the name of the incident with **/purpose** 1. adjust the name of the incident with **/purpose**
2. adjust the description of the incident with **/header** 2. adjust the description of the incident with **/header**
3. choose a front page status (see below) 3. choose a front page status (see below)
4. publish https://status.faelix.net/incident/%(incident_number)s with **@FIH PUBLISH** 4. publish %(incident_url)s with **/incident PUBLISH**
## Status Updates ## Status Updates
* add an update: **@FIH TIMELINE** _text_ * add an update: **/incident TIMELINE** _text_
* set the incident start: **@FIH START** _%(now)s_ (UTC) * set the incident start: **/incident START** _%(now)s_ (UTC)
* set the incident ETR: **@FIH ETR** _%(now)s_ (UTC) * set the incident ETR: **/incident ETR** _%(now)s_ (UTC)
* set the incident resolved time: **@FIH RESOLVED** _%(now)s_ (UTC) * set the incident resolved time: **/incident RESOLVED** _%(now)s_ (UTC)
* set the incident finished time: **@FIH CLOSED** _%(now)s_ (UTC) * set the incident finished time: **/incident CLOSED** _%(now)s_ (UTC)
Again, don't forget to **@FIH PUBLISH** once making changes. Again, don't forget to **/incident PUBLISH** once making changes.
## Front Page Status ## Front Page Status
* **@FIH INCIDENT** * **/incident INCIDENT**
* **@FIH DEGRADED** * **/incident DEGRADED**
* **@FIH MAINTENANCE** * **/incident MAINTENANCE**
* **@FIH NOTICE** * **/incident NOTICE**
* **@FIH OK** * **/incident OK**
Again, after setting the headline status you must: **@FIH PUBLISH** Again, after setting the headline status you must: **/incident PUBLISH**
## Good luck, Incident Commander @%(commander_name)s!""" % { "incident_number": incident_number, ## Good luck, Incident Commander @%(commander_name)s!""" % { "incident_url": raw_incident_number_to_url( incident_number ),
"commander_name": request.form[ 'user_name' ], "commander_name": request.form[ 'user_name' ],
"now": now, "now": now,
}, },
} ) } )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ], app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': 'On behalf of @%s I have started incident ~%s.' % ( request.form[ 'user_name' ], channel_name, ), 'message': 'On behalf of @%s I have started incident ~%s.' % ( request.form[ 'user_name' ], channel_name, ),
} ) } )
return ""
@app.route( '/mattermost/incident', methods = [ 'POST' ] )
def mattermost_incident():
# channel_name: testing
# command: /incident
# channel_id: o3ob1utim3nummoswjyt4174mo
# user_name: marek
# test_domain: faelix
# text: test
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
# token: XXXXXXXXXXXXXXXXXXXXXXXXXX
return "" if request.form[ 'token' ] in app.config[ 'MATTERMOST_COMMAND_TOKENS' ]:
if is_incident_channel_name( request.form[ 'channel_name' ] ):
return mattermost_incident_command()
else:
return mattermost_incident_start()
else: else:
return "Invalid command token." return "Invalid command token."
@ -123,10 +224,10 @@ def exec_to_message( cmdline ):
@app.route( '/mattermost/fih', methods = [ 'POST' ] ) @app.route( '/mattermost/fih', methods = [ 'POST' ] )
def mattermost_fih(): def mattermost_fih():
# text: @FIH PUBLISH # text: /incident PUBLISH
# file_ids: # file_ids:
# user_name: marek # user_name: marek
# trigger_word: @FIH # trigger_word: /incident
# channel_name: incident201806158693 # channel_name: incident201806158693
# timestamp: 1529100914 # timestamp: 1529100914
# channel_id: zq61mk8ig7rmibtbyqf4babhfr # channel_id: zq61mk8ig7rmibtbyqf4babhfr
@ -137,67 +238,7 @@ def mattermost_fih():
# post_id: 8cxn8yeyepyczcfbazdsd1bonw # post_id: 8cxn8yeyepyczcfbazdsd1bonw
if request.form[ 'token' ] in app.config[ 'MATTERMOST_WEBHOOK_TOKENS' ]: if request.form[ 'token' ] in app.config[ 'MATTERMOST_WEBHOOK_TOKENS' ]:
cmd = request.form[ 'text' ].strip().split()[ 1 ].upper() pass
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': "### Executing `" + cmd + "`...",
} )
if cmd == "PUBLISH":
for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
cmdline = app.config[ "MERCURIAL_BIN" ] + " pull " + pull_from
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
cmdline = app.config[ "MERCURIAL_BIN" ] + " merge"
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
cmdline = app.config[ "MERCURIAL_BIN" ] + " commit -m 'merge from " + pull_from + "'"
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
for deploy_to in app.config[ "GROW_DEPLOY_TO" ]:
cmdline = app.config[ "GROW_BIN" ] + " deploy -f " + deploy_to
message = exec_to_message( cmdline )
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': message,
} )
elif cmd == "START":
pass
elif cmd == "ETR":
pass
elif cmd == "RESOLVED":
pass
elif cmd == "CLOSED":
pass
elif cmd == "TIMELINE":
pass
elif cmd == "INCIDENT":
pass
elif cmd == "DEGRADED":
pass
elif cmd == "MAINTENANCE":
pass
elif cmd == "NOTICE":
pass
elif cmd == "OK":
pass
else:
return "WTF command?"
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
'message': "### Finished `" + cmd + "`!",
} )
return "OK"
else: else:
return "Invalid command token." return "Invalid command token."

15
incident.py
Unescape Escape View File

@ -1,15 +0,0 @@
import os
import datetime
import random
import mattermostdriver
random.seed( os.urandom( 8 ) )
def generate_incident_number():
return datetime.datetime.now().strftime( "%Y%m%d" ) + ( "%4d" % int( random.random() * 10000 ) )
class Mattermost( object ):
def __init__( self, *args, **kwargs ):
self.mm = mattermostdriver.Driver( kwargs )
self.mm.login()
Write Preview
Loading…
Cancel
Save