You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
302 lines
13 KiB
Python
302 lines
13 KiB
Python
|
|
from flask import Flask, request
|
|
import os
|
|
import datetime
|
|
import subprocess
|
|
import re
|
|
import random
|
|
import mattermostdriver
|
|
import event
|
|
import slugify
|
|
|
|
random.seed( os.urandom( 8 ) )
|
|
|
|
|
|
class Mattermost( object ):
|
|
def __init__( self, *args, **kwargs ):
|
|
self.mm = mattermostdriver.Driver( kwargs )
|
|
self.mm.login()
|
|
|
|
def strip_ansi( text, escape = re.compile( r'\x1B\[[0-?]*[ -/]*[@-~]' ) ):
|
|
return escape.sub( '', text )
|
|
|
|
def strip_progress( text, escape = re.compile( r'\r[^\n]*\n' ) ):
|
|
return escape.sub( '', text )
|
|
|
|
|
|
app = Flask( __name__ )
|
|
app.config.from_envvar( 'FIH_SETTINGS' )
|
|
|
|
app.mm = Mattermost( url = app.config[ 'MATTERMOST_URL' ],
|
|
port = app.config[ 'MATTERMOST_PORT' ],
|
|
login_id = app.config[ 'MATTERMOST_USER_EMAIL' ],
|
|
password = app.config[ 'MATTERMOST_USER_PASSWORD' ],
|
|
).mm
|
|
|
|
|
|
def generate_raw_incident_number():
|
|
return datetime.datetime.now().strftime( "%Y%m%d" ) + ( "%4d" % int( random.random() * 10000 ) )
|
|
|
|
def raw_incident_number_to_channel_name( incident_number ):
|
|
return app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] + incident_number
|
|
|
|
def raw_incident_number_to_public_incident_number( incident_number ):
|
|
return app.config[ "PUBLIC_INCIDENT_PREFIX" ] + incident_number
|
|
|
|
def raw_incident_number_to_url( incident_number ):
|
|
return app.config[ "INCIDENT_URL_SCHEME" ] % incident_number
|
|
|
|
def is_incident_channel_name( channel_name ):
|
|
return channel_name.startswith( app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] )
|
|
|
|
def channel_name_to_raw_incident_number( channel_name ):
|
|
return channel_name[ len( app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] ): ]
|
|
|
|
|
|
def mattermost_incident_command( command, args, channel_id, raw_incident_number ):
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': "### Executing `" + command + "`...",
|
|
} )
|
|
|
|
if command == "PUBLISH":
|
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " update"
|
|
message = exec_to_message( cmdline )
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
channel = app.mm.channels.get_channel( channel_id )
|
|
slug = slugify.slugify( channel[ 'purpose' ] )
|
|
ev = event.Event( path = app.config[ "STATUS_EVENT_SCHEME" ] % { "incident_number": raw_incident_number,
|
|
"slug": slug,
|
|
},
|
|
glob = app.config[ "STATUS_EVENT_GLOB" ] % { "incident_number": raw_incident_number,
|
|
},
|
|
)
|
|
new = ev.open()
|
|
ev.doc = channel[ 'header' ]
|
|
ev.fields[ '$title@' ] = channel[ 'purpose' ]
|
|
ev.fields[ '$slug' ] = raw_incident_number
|
|
if not ev.fields.get( '$date' ):
|
|
ev.fields[ '$date' ] = {}
|
|
if not ev.fields[ '$date' ].get( 'published' ):
|
|
ev.fields[ '$date' ][ 'published' ] = datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" )
|
|
written = ev.write()
|
|
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': "#### Written `%s`:\n\n```\n%s```" % ( ev.path_short(), written ),
|
|
} )
|
|
|
|
if new:
|
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " add " + ev.path_short()
|
|
message = exec_to_message( cmdline )
|
|
if message:
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " commit -m 'add event for incident " + raw_incident_number + " via FIH' " + ev.path_short()
|
|
message = exec_to_message( cmdline )
|
|
if message:
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
|
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " pull " + pull_from
|
|
message = exec_to_message( cmdline )
|
|
if message:
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " merge"
|
|
message = exec_to_message( cmdline )
|
|
if message:
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
cmdline = app.config[ "MERCURIAL_BIN" ] + " commit -m 'merge from " + pull_from + "'"
|
|
message = exec_to_message( cmdline )
|
|
if message:
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
for deploy_to in app.config[ "GROW_DEPLOY_TO" ]:
|
|
cmdline = app.config[ "GROW_BIN" ] + " deploy -f " + deploy_to
|
|
message = exec_to_message( cmdline )
|
|
if message:
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': message,
|
|
} )
|
|
|
|
elif command == "START":
|
|
pass
|
|
elif command == "ETR":
|
|
pass
|
|
elif command == "RESOLVED":
|
|
pass
|
|
elif command == "CLOSED":
|
|
pass
|
|
elif command == "TIMELINE":
|
|
pass
|
|
elif command == "INCIDENT":
|
|
pass
|
|
elif command == "DEGRADED":
|
|
pass
|
|
elif command == "MAINTENANCE":
|
|
pass
|
|
elif command == "NOTICE":
|
|
pass
|
|
elif command == "OK":
|
|
pass
|
|
else:
|
|
return "WTF command?"
|
|
|
|
app.mm.posts.create_post( options = { 'channel_id': request.form[ 'channel_id' ],
|
|
'message': "### Finished `" + command + "`!",
|
|
} )
|
|
|
|
return "OK"
|
|
|
|
def mattermost_incident_start( description, team_id, user_name, user_id, channel_id ):
|
|
raw_incident_number = incident.generate_raw_incident_number()
|
|
fi_number = raw_incident_number_to_public_incident_number( raw_incident_number )
|
|
channel_name = raw_incident_number_to_channel_name( raw_incident_number )
|
|
|
|
channel = app.mm.channels.create_channel( options = { "team_id": team_id,
|
|
"name": channel_name,
|
|
"display_name": fi_number,
|
|
"purpose": description,
|
|
"header": "",
|
|
"type": "O",
|
|
} )
|
|
|
|
# add user to the channel we just created
|
|
app.mm.client.make_request( 'post', '/channels/' + channel[ 'id' ] + '/members', options = { 'user_id': user_id,
|
|
} )
|
|
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
|
|
'message': "Incident discussion channel for [%s](%s) created by @%s." % ( fi_number, raw_incident_number_to_url( incident_number ), user_name ),
|
|
} )
|
|
|
|
now = datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" )
|
|
|
|
app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
|
|
'message': """\
|
|
# Incident Response Communication Procedure
|
|
|
|
The contents of this channel are strictly confidential.
|
|
|
|
## Public Announcement
|
|
|
|
1. adjust the name of the incident with **/purpose**
|
|
2. adjust the description of the incident with **/header**
|
|
3. choose a front page status (see below)
|
|
4. publish %(incident_url)s with **/incident PUBLISH**
|
|
|
|
## Status Updates
|
|
|
|
* add an update: **/incident TIMELINE** _text_
|
|
* set the incident start: **/incident START** _%(now)s_ (UTC)
|
|
* set the incident ETR: **/incident ETR** _%(now)s_ (UTC)
|
|
* set the incident resolved time: **/incident RESOLVED** _%(now)s_ (UTC)
|
|
* set the incident finished time: **/incident CLOSED** _%(now)s_ (UTC)
|
|
|
|
Again, don't forget to **/incident PUBLISH** once making changes.
|
|
|
|
## Front Page Status
|
|
|
|
* **/incident INCIDENT**
|
|
* **/incident DEGRADED**
|
|
* **/incident MAINTENANCE**
|
|
* **/incident NOTICE**
|
|
* **/incident OK**
|
|
|
|
Again, after setting the headline status you must: **/incident PUBLISH**
|
|
|
|
## Good luck, Incident Commander @%(commander_name)s!""" % { "incident_url": raw_incident_number_to_url( incident_number ),
|
|
"commander_name": user_name,
|
|
"now": now,
|
|
},
|
|
} )
|
|
|
|
app.mm.posts.create_post( options = { 'channel_id': channel_id,
|
|
'message': 'On behalf of @%s I have started incident ~%s.' % ( request.form[ 'user_name' ], channel_name, ),
|
|
} )
|
|
|
|
return ""
|
|
|
|
@app.route( '/mattermost/incident', methods = [ 'POST' ] )
|
|
def mattermost_incident():
|
|
# channel_name: testing
|
|
# command: /incident
|
|
# channel_id: o3ob1utim3nummoswjyt4174mo
|
|
# user_name: marek
|
|
# test_domain: faelix
|
|
# text: test
|
|
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
|
|
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
|
|
# token: XXXXXXXXXXXXXXXXXXXXXXXXXX
|
|
|
|
if request.form[ 'token' ] in app.config[ 'MATTERMOST_COMMAND_TOKENS' ]:
|
|
if is_incident_channel_name( request.form[ 'channel_name' ] ):
|
|
text = request.form[ 'text' ].strip().split()
|
|
raw_incident_number = channel_name_to_raw_incident_number( request.form[ 'channel_name' ] )
|
|
|
|
return mattermost_incident_command( command = text[ 0 ].upper(),
|
|
args = " ".join( text[ 1: ] ),
|
|
channel_id = request.form[ 'channel_id' ],
|
|
raw_incident_number = raw_incident_number,
|
|
)
|
|
else:
|
|
return mattermost_incident_start( description = request.form[ 'text' ],
|
|
team_id = request.form[ 'team_id' ],
|
|
user_name = request.form[ 'user_name' ],
|
|
user_id = request.form[ 'user_id' ],
|
|
channel_id = request.form[ 'channel_id' ],
|
|
)
|
|
else:
|
|
return "Invalid command token."
|
|
|
|
|
|
def exec_to_message( cmdline ):
|
|
results = "#### `" + cmdline + "`:\n\n"
|
|
process = subprocess.Popen( cmdline, shell = True, cwd = app.config[ "STATUS_ROOT" ], stdout = subprocess.PIPE, stderr = subprocess.PIPE )
|
|
|
|
stdout = strip_progress( strip_ansi( process.stdout.read().decode( 'ascii', 'ignore' ) ) )
|
|
if stdout:
|
|
results += "\n\n```\n" + stdout.replace( "```", "` ` `" ) + "```\n"
|
|
stderr = strip_progress( strip_ansi( process.stderr.read().decode( 'ascii', 'ignore' ) ) )
|
|
if stderr:
|
|
results += "\n\n```\n" + stderr.replace( "```", "` ` `" ) + "```\n"
|
|
if not stdout and not stderr:
|
|
results += '_no output_\n'
|
|
return results
|
|
|
|
|
|
@app.route( '/mattermost/fih', methods = [ 'POST' ] )
|
|
def mattermost_fih():
|
|
# text: /incident PUBLISH
|
|
# file_ids:
|
|
# user_name: marek
|
|
# trigger_word: /incident
|
|
# channel_name: incident201806158693
|
|
# timestamp: 1529100914
|
|
# channel_id: zq61mk8ig7rmibtbyqf4babhfr
|
|
# team_id: 3woo8mbjrbb1in53xwdzi4ynqh
|
|
# token: nn54y4ozbtnzb8pri3c4gdhbce
|
|
# user_id: ko3uxmend7ne8ger4uw9mxkt1o
|
|
# team_domain: faelix
|
|
# post_id: 8cxn8yeyepyczcfbazdsd1bonw
|
|
|
|
if request.form[ 'token' ] in app.config[ 'MATTERMOST_WEBHOOK_TOKENS' ]:
|
|
pass
|
|
else:
|
|
return "Invalid command token."
|
|
|
|
@app.route( '/' )
|
|
def hello_world():
|
|
return 'Hello, World!'
|