incident_handler/fih.py

from flask import Flask, request
import os
import datetime
import subprocess
import re
import random
import mattermostdriver
import event
import slugify

random.seed( os.urandom( 8 ) )


class Mattermost( object ):
    def __init__( self, *args, **kwargs ):
        self.mm = mattermostdriver.Driver( kwargs )
        self.mm.login()

def strip_ansi( text, escape = re.compile( r'\x1B\[[0-?]*[ -/]*[@-~]' ) ):
    return escape.sub( '', text )

def strip_progress( text, escape = re.compile( r'\r[^\n]*\n' ) ):
    return escape.sub( '', text )


app = Flask( __name__ )
app.config.from_envvar( 'FIH_SETTINGS' )

app.mm = Mattermost( url = app.config[ 'MATTERMOST_URL' ],
                     port = app.config[ 'MATTERMOST_PORT' ],
                     login_id = app.config[ 'MATTERMOST_USER_EMAIL' ],
                     password = app.config[ 'MATTERMOST_USER_PASSWORD' ],
                     ).mm


def generate_raw_incident_number():
    return datetime.datetime.now().strftime( "%Y%m%d" ) + ( "%4d" % int( random.random() * 10000 ) )

def raw_incident_number_to_channel_name( incident_number ):
    return app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] + incident_number

def raw_incident_number_to_public_incident_number( incident_number ):
    return app.config[ "PUBLIC_INCIDENT_PREFIX" ] + incident_number

def raw_incident_number_to_url( incident_number ):
    return app.config[ "INCIDENT_URL_SCHEME" ] % incident_number

def is_incident_channel_name( channel_name ):
    return channel_name.startswith( app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] )

def channel_name_to_raw_incident_number( channel_name ):
    return channel_name[ len( app.config[ "MATTERMOST_INCIDENT_CHANNEL_PREFIX" ] ): ]


def mattermost_incident_command( command, args, channel_id, raw_incident_number, user_name, user_id ):
    channel = app.mm.channels.get_channel( channel_id )
    slug = slugify.slugify( channel[ 'purpose' ] )
    index = event.Index( path = app.config[ "STATUS_EVENT_INDEX" ] )
    ev = event.Event( path = app.config[ "STATUS_EVENT_SCHEME" ] % { "incident_number": raw_incident_number,
                                                                     "slug": slug,
                                                                     },
                      glob = app.config[ "STATUS_EVENT_GLOB" ] % { "incident_number": raw_incident_number,
                                                                   },
                      )

    if command == "PUBLISH":
        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "### `PUBLISH` requested by @%s!" % user_name,
                                              } )

        private_channel = app.mm.channels.create_direct_message_channel( [ app.config[ "MATTERMOST_USER_ID" ], user_id ] )
        private_channel_id = private_channel[ 'id' ]

        # update local repository
        cmdline = app.config[ "MERCURIAL_BIN" ] + " update"
        message = exec_to_message( cmdline )
        app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                              'message': message,
                                              } )

        # create/update event content file
        new = ev.open()
        ev.doc = channel[ 'header' ]
        ev.set_title( channel[ 'purpose' ] )
        ev.set_slug( raw_incident_number )
        ev.set_published( datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" ) )
        ev.set_year( datetime.datetime.utcnow().strftime( "%Y" ) )
        written = ev.write()

        short_written = written
        if len( short_written ) > 512:
            short_written = short_written[ :509 ] + "..."
        app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                              'message': "#### Written `%s`:\n\n```\n%s```" % ( ev.path_short(), short_written ),
                                              } )

        cmdline = app.config[ "MERCURIAL_BIN" ] + " add " + ev.path_short()
        message = exec_to_message( cmdline )
        if message:
            app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                  'message': message,
                                                  } )

        cmdline = app.config[ "MERCURIAL_BIN" ] + " commit -m 'event for incident " + raw_incident_number + " via FIH' " + ev.path_short()
        message = exec_to_message( cmdline )
        if message:
            app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                  'message': message,
                                                  } )

        # pull, merge, commit
        for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
            cmdline = app.config[ "MERCURIAL_BIN" ] + " pull " + pull_from
            message = exec_to_message( cmdline )
            if message:
                app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                      'message': message,
                                                      } )

            cmdline = app.config[ "MERCURIAL_BIN" ] + " merge"
            message = exec_to_message( cmdline )
            if message:
                app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                      'message': message,
                                                      } )

            cmdline = app.config[ "MERCURIAL_BIN" ] + " commit -m 'merge from " + pull_from + "'"
            message = exec_to_message( cmdline )
            if message:
                app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                      'message': message,
                                                      } )

        # push updated repository            
        for pull_from in app.config[ "MERCURIAL_PUSH_TO" ]:
            cmdline = app.config[ "MERCURIAL_BIN" ] + " push " + pull_from
            message = exec_to_message( cmdline )
            if message:
                app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                      'message': message,
                                                      } )

        for deploy_to in app.config[ "GROW_DEPLOY_TO" ]:
            cmdline = app.config[ "GROW_BIN" ] + " deploy -f " + deploy_to
            message = exec_to_message( cmdline )
            if message:
                app.mm.posts.create_post( options = { 'channel_id': private_channel_id,
                                                      'message': message,
                                                      } )

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "### `PUBLISH` completed!",
                                              } )

    elif command == "START":
        ev.open()
        started = ev.set_started( args or datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" ) )
        ev.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Event start date set by %s to: `%s`" % ( user_name, started ),
                                              } )
    elif command == "ETR":
        ev.open()
        expected = ev.set_expected( args or datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" ) )
        ev.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Event estimated time of resolution set by %s to: `%s`" % ( user_name, expected ),
                                              } )
    elif command == "FINISHED":
        ev.open()
        finished = ev.set_finished( args or datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" ) )
        ev.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Event finished time set by %s to: `%s`" % ( user_name, finished ),
                                              } )
    elif command == "RESOLVED":
        ev.open()
        resolved = ev.set_resolved( args or datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" ) )
        ev.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Event resolved time set by %s to: `%s`" % ( user_name, resolved ),
                                              } )
    elif command == "TIMELINE":
        if not args:
            return "You need to specify some text to add to the timeline..."

        ev.open()
        ev.add_timeline( time = datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" ), line = args )
        ev.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Timeline entry added by @%s:\n\n> %s" % ( user_name, args ),
                                              } )

    elif command == "INCIDENT":
        ev.open()
        flags = ev.toggle_incident()
        ev.write()
        index.open()
        index.set_flags( flags.split() )
        index.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Flags set by @%s to: `%s`" % ( user_name, flags ),
                                              } )
    elif command == "DEGRADED":
        ev.open()
        flags = ev.toggle_degraded()
        ev.write()
        index.open()
        index.set_flags( flags.split() )
        index.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Flags set by @%s to: `%s`" % ( user_name, flags ),
                                              } )
    elif command == "MAINTENANCE":
        ev.open()
        flags = ev.toggle_maintenance()
        ev.write()
        index.open()
        index.set_flags( flags.split() )
        index.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Flags set by @%s to: `%s`" % ( user_name, flags ),
                                              } )
    elif command == "NOTICE":
        ev.open()
        flags = ev.toggle_notice()
        ev.write()
        index.open()
        index.set_flags( flags.split() )
        index.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Flags set by @%s to: `%s`" % ( user_name, flags ),
                                              } )
    elif command == "OK":
        ev.open()
        flags = ev.toggle_ok()
        ev.write()
        index.open()
        index.set_flags( flags.split() )
        index.write()

        app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                              'message': "Flags set by @%s to: `%s`" % ( user_name, flags ),
                                              } )
    else:
        post_help( channel, raw_incident_number, user_name )

    return ""

def post_help( channel, raw_incident_number, user_name ):
    now = datetime.datetime.utcnow().strftime( "%Y-%m-%d %H:%M" )

    app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
                                          'message': """\
# Incident Response Communication Procedure

The contents of this channel are strictly confidential.

## Public Announcement

1. adjust the name of the incident with **/purpose**
2. adjust the description of the incident with **/header**
3. choose a front page status (see below)
4. publish %(incident_url)s with **/incident PUBLISH**

## Status Updates

* add an update: **/incident TIMELINE** _text_
* set the event start: **/incident START** _%(now)s_ (UTC)
* set the event ETR: **/incident ETR** _%(now)s_ (UTC)
* set the event finished time: **/incident FINISHED** _%(now)s_ (UTC)
* set the event closure time: **/incident RESOLVED** _%(now)s_ (UTC)

Again, don't forget to **/incident PUBLISH** once making changes.

## Front Page Status

* **/incident [ INCIDENT | DEGRADED | MAINTENANCE | NOTICE | OK ]**

Again, after setting the headline status you must: **/incident PUBLISH**

## Good luck, Incident Commander @%(commander_name)s!""" % { "incident_url": raw_incident_number_to_url( raw_incident_number ),
                                                             "commander_name": user_name,
                                                             "now": now,
                                                             },
                                          } )


def mattermost_incident_start( description, team_id, user_name, user_id, channel_id ):
    raw_incident_number = generate_raw_incident_number()
    fi_number = raw_incident_number_to_public_incident_number( raw_incident_number )
    channel_name = raw_incident_number_to_channel_name( raw_incident_number )

    channel = app.mm.channels.create_channel( options = { "team_id": team_id,
                                                          "name": channel_name,
                                                          "display_name": fi_number,
                                                          "purpose": description,
                                                          "header": "",
                                                          "type": "O",
                                                          } )

    # add user to the channel we just created
    app.mm.client.make_request( 'post', '/channels/' + channel[ 'id' ] + '/members', options = { 'user_id': user_id,
                                                                                                 } )
    app.mm.posts.create_post( options = { 'channel_id': channel[ 'id' ],
                                          'message': "Incident discussion channel for [%s](%s) created by @%s." % ( fi_number, raw_incident_number_to_url( raw_incident_number ), user_name ),
                                          } )

    post_help( channel, raw_incident_number, user_name )

    app.mm.posts.create_post( options = { 'channel_id': channel_id,
                                          'message': 'On behalf of @%s I have started incident ~%s.' % ( user_name, channel_name, ),
                                          } )

    return ""

@app.route( '/mattermost/incident', methods = [ 'POST' ] )
def mattermost_incident():
    # channel_name: testing
    # command: /incident
    # channel_id: o3ob1utim3nummoswjyt4174mo
    # user_name: marek
    # test_domain: faelix
    # text: test
    # team_id: 3woo8mbjrbb1in53xwdzi4ynqh
    # user_id: ko3uxmend7ne8ger4uw9mxkt1o
    # token: XXXXXXXXXXXXXXXXXXXXXXXXXX

    if request.form[ 'token' ] in app.config[ 'MATTERMOST_COMMAND_TOKENS' ]:
        if is_incident_channel_name( request.form[ 'channel_name' ] ):
            text = request.form[ 'text' ].strip().split()
            raw_incident_number = channel_name_to_raw_incident_number( request.form[ 'channel_name' ] )

            return mattermost_incident_command( command = text[ 0 ].upper(),
                                                args = " ".join( text[ 1: ] ),
                                                channel_id = request.form[ 'channel_id' ],
                                                user_name = request.form[ 'user_name' ],
                                                user_id = request.form[ 'user_id' ],
                                                raw_incident_number = raw_incident_number,
                                                )
        else:
            return mattermost_incident_start( description = request.form[ 'text' ],
                                              team_id = request.form[ 'team_id' ],
                                              user_name = request.form[ 'user_name' ],
                                              user_id = request.form[ 'user_id' ],
                                              channel_id = request.form[ 'channel_id' ],
                                              )
    else:
        return "Invalid command token."


def exec_to_message( cmdline ):
    results = "#### `" + cmdline + "`:\n\n"
    process = subprocess.Popen( cmdline, shell = True, cwd = app.config[ "STATUS_ROOT" ], stdout = subprocess.PIPE, stderr = subprocess.PIPE )

    stdout = strip_progress( strip_ansi( process.stdout.read().decode( 'ascii', 'ignore' ) ) )
    if stdout:
        results += "\n\n`````````\n" + stdout + "`````````\n"
    stderr = strip_progress( strip_ansi( process.stderr.read().decode( 'ascii', 'ignore' ) ) )
    if stderr:
        results += "\n\n`````````\n" + stderr + "`````````\n"
    if not stdout and not stderr:
        results += '_no output_\n'
    return results


@app.route( '/mattermost/fih', methods = [ 'POST' ] )
def mattermost_fih():
    # text: /incident PUBLISH
    # file_ids:
    # user_name: marek
    # trigger_word: /incident
    # channel_name: incident201806158693
    # timestamp: 1529100914
    # channel_id: zq61mk8ig7rmibtbyqf4babhfr
    # team_id: 3woo8mbjrbb1in53xwdzi4ynqh
    # token: nn54y4ozbtnzb8pri3c4gdhbce
    # user_id: ko3uxmend7ne8ger4uw9mxkt1o
    # team_domain: faelix
    # post_id: 8cxn8yeyepyczcfbazdsd1bonw

    if request.form[ 'token' ] in app.config[ 'MATTERMOST_WEBHOOK_TOKENS' ]:
        pass
    else:
        return "Invalid command token."
    
@app.route( '/' )
def hello_world():
    return 'Hello, World!'