massive restructure
This commit is contained in:
parent
82126a1f51
commit
27afe2229f
259
fih.py
259
fih.py
@ -5,7 +5,16 @@ import incident
|
|||||||
import datetime
|
import datetime
|
||||||
import subprocess
|
import subprocess
|
||||||
import re
|
import re
|
||||||
|
import random
|
||||||
|
import mattermostdriver
|
||||||
|
|
||||||
|
random.seed( os.urandom( 8 ) )
|
||||||
|
|
||||||
|
|
||||||
|
class Mattermost( object ):
|
||||||
|
def __init__( self, *args, **kwargs ):
|
||||||
|
self.mm = mattermostdriver.Driver( kwargs )
|
||||||
|
self.mm.login()
|
||||||
|
|
||||||
def strip_ansi( text, escape = re.compile( r'\x1B\[[0-?]*[ -/]*[@-~]' ) ):
|
def strip_ansi( text, escape = re.compile( r'\x1B\[[0-?]*[ -/]*[@-~]' ) ):
|
||||||
return escape.sub( '', text )
|
return escape.sub( '', text )
|
||||||
@ -17,133 +26,43 @@ def strip_progress( text, escape = re.compile( r'\r[^\n]*\n' ) ):
|
|||||||
app = Flask( __name__ )
|
app = Flask( __name__ )
|
||||||
app.config.from_envvar( 'FIH_SETTINGS' )
|
app.config.from_envvar( 'FIH_SETTINGS' )
|
||||||
|
|
||||||
app.mm = incident.Mattermost( url = app.config[ 'MATTERMOST_URL' ],
|
app.mm = Mattermost( url = app.config[ 'MATTERMOST_URL' ],
|
||||||
port = app.config[ 'MATTERMOST_PORT' ],
|
port = app.config[ 'MATTERMOST_PORT' ],
|
||||||
login_id = app.config[ 'MATTERMOST_USER_EMAIL' ],
|
login_id = app.config[ 'MATTERMOST_USER_EMAIL' ],
|
||||||
password = app.config[ 'MATTERMOST_USER_PASSWORD' ],
|
password = app.config[ 'MATTERMOST_USER_PASSWORD' ],
|
||||||
).mm
|
).mm
|
||||||
|
|
||||||
@app.route( '/mattermost/incident', methods = [ 'POST' ] )
|
|
||||||
def mattermost_incident():
|
|
||||||
# channel_name: testing
|
|
||||||
# command: /incident
|
|
||||||
# channel_id: o3ob1utim3nummoswjyt4174mo
|
|
||||||
# user_name: marek
|
|
||||||
# test_domain: faelix
|
|
||||||
# text: test
|
|
||||||
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
|
|
||||||
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
|
|
||||||
# token: XXXXXXXXXXXXXXXXXXXXXXXXXX
|
|
||||||
|
|
||||||
if request.form[ 'token' ] in app.config[ 'MATTERMOST_COMMAND_TOKENS' ]:
|
def generate_raw_incident_number():
|
||||||
incident_number = incident.generate_incident_number()
|
return datetime.datetime.now().strftime( "%Y%m%d" ) + ( "%4d" % int( random.random() * 10000 ) )
|
||||||
fi_number = "FI#" + incident_number
|
|
||||||
channel_name = "incident" + incident_number
|
|
||||||
purpose = request.form[ 'text' ]
|
|
||||||
|
|
||||||
channel = app.mm.channels.create_channel( options = { "team_id": request.form[ 'team_id' ],
|
def raw_incident_number_to_channel_name( incident_number ):
|
||||||
"name": channel_name,
|
return app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] + incident_number
|
||||||
"display_name": fi_number,
|
|
||||||
"purpose": purpose,
|
|
||||||
"header": "",
|
|
||||||
"type": "O",
|
|
||||||
} )
|
|
||||||
|
|
||||||
# add user to the channel we just created
|
def raw_incident_number_to_public_incident_number( incident_number ):
|
||||||
app.mm.client.make_request( 'post', '/channels/' + channel[ 'id' ] + '/members', options = { 'user_id': request.form[ 'user_id' ],
|
return app.config[ "PUBLIC_INCIDENT_PREFIX" ] + incident_number
|
||||||
} )
|
|
||||||
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
|
|
||||||
'message': "Incident discussion channel for [%s](https://status.faelix.net/incident/%s) created by @%s." % ( fi_number, incident_number, request.form[ 'user_name' ] ),
|
|
||||||
} )
|
|
||||||
|
|
||||||
now = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S" )
|
def raw_incident_number_to_url( incident_number ):
|
||||||
|
return app.config[ "INCIDENT_URL_SCHEME" ] % incident_number
|
||||||
|
|
||||||
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
|
def is_incident_channel_name( channel_name ):
|
||||||
'message': """\
|
return channel_name.startswith( app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] )
|
||||||
# Incident Response Communication Procedure
|
|
||||||
|
|
||||||
The contents of this channel are strictly confidential.
|
|
||||||
|
|
||||||
## Public Announcement
|
|
||||||
|
|
||||||
1. adjust the name of the incident with **/purpose**
|
|
||||||
2. adjust the description of the incident with **/header**
|
|
||||||
3. choose a front page status (see below)
|
|
||||||
4. publish https://status.faelix.net/incident/%(incident_number)s with **@FIH PUBLISH**
|
|
||||||
|
|
||||||
## Status Updates
|
|
||||||
|
|
||||||
* add an update: **@FIH TIMELINE** _text_
|
|
||||||
* set the incident start: **@FIH START** _%(now)s_ (UTC)
|
|
||||||
* set the incident ETR: **@FIH ETR** _%(now)s_ (UTC)
|
|
||||||
* set the incident resolved time: **@FIH RESOLVED** _%(now)s_ (UTC)
|
|
||||||
* set the incident finished time: **@FIH CLOSED** _%(now)s_ (UTC)
|
|
||||||
|
|
||||||
Again, don't forget to **@FIH PUBLISH** once making changes.
|
|
||||||
|
|
||||||
## Front Page Status
|
|
||||||
|
|
||||||
* **@FIH INCIDENT**
|
|
||||||
* **@FIH DEGRADED**
|
|
||||||
* **@FIH MAINTENANCE**
|
|
||||||
* **@FIH NOTICE**
|
|
||||||
* **@FIH OK**
|
|
||||||
|
|
||||||
Again, after setting the headline status you must: **@FIH PUBLISH**
|
|
||||||
|
|
||||||
## Good luck, Incident Commander @%(commander_name)s!""" % { "incident_number": incident_number,
|
|
||||||
"commander_name": request.form[ 'user_name' ],
|
|
||||||
"now": now,
|
|
||||||
},
|
|
||||||
} )
|
|
||||||
|
|
||||||
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
|
|
||||||
'message': 'On behalf of @%s I have started incident ~%s.' % ( request.form[ 'user_name' ], channel_name, ),
|
|
||||||
} )
|
|
||||||
|
|
||||||
return ""
|
|
||||||
else:
|
|
||||||
return "Invalid command token."
|
|
||||||
|
|
||||||
|
|
||||||
def exec_to_message( cmdline ):
|
def mattermost_incident_command():
|
||||||
results = "#### `" + cmdline + "`:\n\n"
|
cmd = request.form[ 'text' ].strip().split()[ 0 ].upper()
|
||||||
process = subprocess.Popen( cmdline, shell = True, cwd = app.config[ "STATUS_ROOT" ], stdout = subprocess.PIPE, stderr = subprocess.PIPE )
|
|
||||||
|
|
||||||
stdout = strip_progress( strip_ansi( process.stdout.read().decode( 'ascii', 'ignore' ) ) )
|
|
||||||
if stdout:
|
|
||||||
results += "\n\n```\n" + stdout.replace( "```", "` ` `" ) + "```\n"
|
|
||||||
stderr = strip_progress( strip_ansi( process.stderr.read().decode( 'ascii', 'ignore' ) ) )
|
|
||||||
if stderr:
|
|
||||||
results += "\n\n```\n" + stderr.replace( "```", "` ` `" ) + "```\n"
|
|
||||||
if not stdout and not stderr:
|
|
||||||
results += '_no output_\n'
|
|
||||||
return results
|
|
||||||
|
|
||||||
|
|
||||||
@app.route( '/mattermost/fih', methods = [ 'POST' ] )
|
|
||||||
def mattermost_fih():
|
|
||||||
# text: @FIH PUBLISH
|
|
||||||
# file_ids:
|
|
||||||
# user_name: marek
|
|
||||||
# trigger_word: @FIH
|
|
||||||
# channel_name: incident201806158693
|
|
||||||
# timestamp: 1529100914
|
|
||||||
# channel_id: zq61mk8ig7rmibtbyqf4babhfr
|
|
||||||
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
|
|
||||||
# token: nn54y4ozbtnzb8pri3c4gdhbce
|
|
||||||
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
|
|
||||||
# team_domain: faelix
|
|
||||||
# post_id: 8cxn8yeyepyczcfbazdsd1bonw
|
|
||||||
|
|
||||||
if request.form[ 'token' ] in app.config[ 'MATTERMOST_WEBHOOK_TOKENS' ]:
|
|
||||||
cmd = request.form[ 'text' ].strip().split()[ 1 ].upper()
|
|
||||||
|
|
||||||
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
|
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
|
||||||
'message': "### Executing `" + cmd + "`...",
|
'message': "### Executing `" + cmd + "`...",
|
||||||
} )
|
} )
|
||||||
|
|
||||||
if cmd == "PUBLISH":
|
if cmd == "PUBLISH":
|
||||||
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " update"
|
||||||
|
message = exec_to_message( cmdline )
|
||||||
|
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
|
||||||
|
'message': message,
|
||||||
|
} )
|
||||||
|
|
||||||
for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
|
for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
|
||||||
cmdline = app.config[ "MERCURIAL_BIN" ] + " pull " + pull_from
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " pull " + pull_from
|
||||||
message = exec_to_message( cmdline )
|
message = exec_to_message( cmdline )
|
||||||
@ -198,6 +117,128 @@ def mattermost_fih():
|
|||||||
} )
|
} )
|
||||||
|
|
||||||
return "OK"
|
return "OK"
|
||||||
|
|
||||||
|
def mattermost_incident_start():
|
||||||
|
raw_incident_number = incident.generate_raw_incident_number()
|
||||||
|
fi_number = raw_incident_number_to_public_incident_number( raw_incident_number )
|
||||||
|
channel_name = raw_incident_number_to_channel_name( raw_incident_number )
|
||||||
|
purpose = request.form[ 'text' ]
|
||||||
|
|
||||||
|
channel = app.mm.channels.create_channel( options = { "team_id": request.form[ 'team_id' ],
|
||||||
|
"name": channel_name,
|
||||||
|
"display_name": fi_number,
|
||||||
|
"purpose": purpose,
|
||||||
|
"header": "",
|
||||||
|
"type": "O",
|
||||||
|
} )
|
||||||
|
|
||||||
|
# add user to the channel we just created
|
||||||
|
app.mm.client.make_request( 'post', '/channels/' + channel[ 'id' ] + '/members', options = { 'user_id': request.form[ 'user_id' ],
|
||||||
|
} )
|
||||||
|
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
|
||||||
|
'message': "Incident discussion channel for [%s](%s) created by @%s." % ( fi_number, raw_incident_number_to_url( incident_number ), request.form[ 'user_name' ] ),
|
||||||
|
} )
|
||||||
|
|
||||||
|
now = datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" )
|
||||||
|
|
||||||
|
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
|
||||||
|
'message': """\
|
||||||
|
# Incident Response Communication Procedure
|
||||||
|
|
||||||
|
The contents of this channel are strictly confidential.
|
||||||
|
|
||||||
|
## Public Announcement
|
||||||
|
|
||||||
|
1. adjust the name of the incident with **/purpose**
|
||||||
|
2. adjust the description of the incident with **/header**
|
||||||
|
3. choose a front page status (see below)
|
||||||
|
4. publish %(incident_url)s with **/incident PUBLISH**
|
||||||
|
|
||||||
|
## Status Updates
|
||||||
|
|
||||||
|
* add an update: **/incident TIMELINE** _text_
|
||||||
|
* set the incident start: **/incident START** _%(now)s_ (UTC)
|
||||||
|
* set the incident ETR: **/incident ETR** _%(now)s_ (UTC)
|
||||||
|
* set the incident resolved time: **/incident RESOLVED** _%(now)s_ (UTC)
|
||||||
|
* set the incident finished time: **/incident CLOSED** _%(now)s_ (UTC)
|
||||||
|
|
||||||
|
Again, don't forget to **/incident PUBLISH** once making changes.
|
||||||
|
|
||||||
|
## Front Page Status
|
||||||
|
|
||||||
|
* **/incident INCIDENT**
|
||||||
|
* **/incident DEGRADED**
|
||||||
|
* **/incident MAINTENANCE**
|
||||||
|
* **/incident NOTICE**
|
||||||
|
* **/incident OK**
|
||||||
|
|
||||||
|
Again, after setting the headline status you must: **/incident PUBLISH**
|
||||||
|
|
||||||
|
## Good luck, Incident Commander @%(commander_name)s!""" % { "incident_url": raw_incident_number_to_url( incident_number ),
|
||||||
|
"commander_name": request.form[ 'user_name' ],
|
||||||
|
"now": now,
|
||||||
|
},
|
||||||
|
} )
|
||||||
|
|
||||||
|
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
|
||||||
|
'message': 'On behalf of @%s I have started incident ~%s.' % ( request.form[ 'user_name' ], channel_name, ),
|
||||||
|
} )
|
||||||
|
|
||||||
|
return ""
|
||||||
|
|
||||||
|
@app.route( '/mattermost/incident', methods = [ 'POST' ] )
|
||||||
|
def mattermost_incident():
|
||||||
|
# channel_name: testing
|
||||||
|
# command: /incident
|
||||||
|
# channel_id: o3ob1utim3nummoswjyt4174mo
|
||||||
|
# user_name: marek
|
||||||
|
# test_domain: faelix
|
||||||
|
# text: test
|
||||||
|
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
|
||||||
|
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
|
||||||
|
# token: XXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||||
|
|
||||||
|
if request.form[ 'token' ] in app.config[ 'MATTERMOST_COMMAND_TOKENS' ]:
|
||||||
|
if is_incident_channel_name( request.form[ 'channel_name' ] ):
|
||||||
|
return mattermost_incident_command()
|
||||||
|
else:
|
||||||
|
return mattermost_incident_start()
|
||||||
|
else:
|
||||||
|
return "Invalid command token."
|
||||||
|
|
||||||
|
|
||||||
|
def exec_to_message( cmdline ):
|
||||||
|
results = "#### `" + cmdline + "`:\n\n"
|
||||||
|
process = subprocess.Popen( cmdline, shell = True, cwd = app.config[ "STATUS_ROOT" ], stdout = subprocess.PIPE, stderr = subprocess.PIPE )
|
||||||
|
|
||||||
|
stdout = strip_progress( strip_ansi( process.stdout.read().decode( 'ascii', 'ignore' ) ) )
|
||||||
|
if stdout:
|
||||||
|
results += "\n\n```\n" + stdout.replace( "```", "` ` `" ) + "```\n"
|
||||||
|
stderr = strip_progress( strip_ansi( process.stderr.read().decode( 'ascii', 'ignore' ) ) )
|
||||||
|
if stderr:
|
||||||
|
results += "\n\n```\n" + stderr.replace( "```", "` ` `" ) + "```\n"
|
||||||
|
if not stdout and not stderr:
|
||||||
|
results += '_no output_\n'
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
@app.route( '/mattermost/fih', methods = [ 'POST' ] )
|
||||||
|
def mattermost_fih():
|
||||||
|
# text: /incident PUBLISH
|
||||||
|
# file_ids:
|
||||||
|
# user_name: marek
|
||||||
|
# trigger_word: /incident
|
||||||
|
# channel_name: incident201806158693
|
||||||
|
# timestamp: 1529100914
|
||||||
|
# channel_id: zq61mk8ig7rmibtbyqf4babhfr
|
||||||
|
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
|
||||||
|
# token: nn54y4ozbtnzb8pri3c4gdhbce
|
||||||
|
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
|
||||||
|
# team_domain: faelix
|
||||||
|
# post_id: 8cxn8yeyepyczcfbazdsd1bonw
|
||||||
|
|
||||||
|
if request.form[ 'token' ] in app.config[ 'MATTERMOST_WEBHOOK_TOKENS' ]:
|
||||||
|
pass
|
||||||
else:
|
else:
|
||||||
return "Invalid command token."
|
return "Invalid command token."
|
||||||
|
|
||||||
|
15
incident.py
15
incident.py
@ -1,15 +0,0 @@
|
|||||||
|
|
||||||
import os
|
|
||||||
import datetime
|
|
||||||
import random
|
|
||||||
import mattermostdriver
|
|
||||||
|
|
||||||
random.seed( os.urandom( 8 ) )
|
|
||||||
|
|
||||||
def generate_incident_number():
|
|
||||||
return datetime.datetime.now().strftime( "%Y%m%d" ) + ( "%4d" % int( random.random() * 10000 ) )
|
|
||||||
|
|
||||||
class Mattermost( object ):
|
|
||||||
def __init__( self, *args, **kwargs ):
|
|
||||||
self.mm = mattermostdriver.Driver( kwargs )
|
|
||||||
self.mm.login()
|
|
Loading…
x
Reference in New Issue
Block a user