#!/usr/bin/python from fulcrmpy import apiv2 as fulcrm import sys import requests from requests.auth import HTTPBasicAuth import datetime import dns.resolver import json import re resolver = dns.resolver.Resolver() vlan_re = re.compile( r"([:\.]?[0-9]+)" ) THINGTYPE_VM = "/api/v2/thingtype/4/" USER_MAREK = "/api/v2/user/1/" ganeti_cluster = sys.argv[ 1 ] fulcrmapi = fulcrm.APIv2( sys.argv[ 2 ], sys.argv[ 3 ] ) ganeti_auth = HTTPBasicAuth( sys.argv[ 4 ], sys.argv[ 5 ] ) def ganeti_get_instances( cluster ): global ganeti_auth cluster = "127.0.0.1" req = requests.get( "https://" + cluster + ":5080/2/instances", verify = False, auth = ganeti_auth ) return req.json() def ganeti_get_instance( cluster, name ): global ganeti_auth cluster = "127.0.0.1" req = requests.get( "https://" + cluster + ":5080/2/instances/" + name, verify = False, auth = ganeti_auth ) return req.json() def fulcrm_get_thing( uuid = None, source_str = None, source = None, name = None ): if uuid: for thing in fulcrmapi.get_many( "/api/v2/thing/", query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], "passport__system": [ "gan2ful.%s" % ganeti_cluster ], "passport__passport": [ uuid ], } ): return ( thing, True ) for thing in fulcrmapi.get_many( "/api/v2/thing/", query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], "uuid": [ uuid ], } ): return ( thing, False ) if source_str: if source: for thing in fulcrmapi.get_many( "/api/v2/thing/", query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], "source": [ source ], "source_str": [ source_str ], } ): return ( thing, False ) else: for thing in fulcrmapi.get_many( "/api/v2/thing/", query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], "source_str": [ source_str ], } ): return ( thing, False ) if name: for thing in fulcrmapi.get_many( "/api/v2/thing/", query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], "name": [ name ], } ): if thing[ 'source' ]: print( '"%s" already exists source "%s" source_str "%s"' % ( name, thing[ 'source' ], thing[ 'source_str' ] ) ) else: return ( thing, False ) def ganeti_vlan( val ): vlans = { "untagged": None, "tagged": [] } if val: for vid in [ x for ( i, x ) in enumerate( vlan_re.split( val ) ) if i % 2 ]: if vid.startswith( ":" ): vlans[ 'tagged' ].append( int( vid[ 1: ] ) ) elif vid.startswith( "." ): vlans[ 'untagged' ] = int( vid[ 1: ] ) else: vlans[ 'untagged' ] = int( vid ) return vlans def unix_to_8601( t ): if t: return datetime.datetime.utcfromtimestamp( t ).strftime( "%Y-%m-%dT%H:%M:%SZ" ) def fulcrm_set_d( d, key, value ): logs = [] if key in d: if d[ key ] != value: logs.append( "changing %s from %s" % ( repr( key ), json.dumps( value ) ) ) d[ key ] = value else: d[ key ] = value logs.append( "adding value for %s" % ( repr( key ), ) ) return logs if __name__ == '__main__': fulcrm_thing_cluster = fulcrm_get_thing( source = "ganeti-cluster", source_str = ganeti_cluster, name = ganeti_cluster ) if not fulcrm_thing_cluster: raise ValueError( "cannot find %s cluster in fulcrm" % ganeti_cluster ) for instance in ganeti_get_instances( ganeti_cluster ): instance_name = instance[ 'id' ] instance_data = ganeti_get_instance( ganeti_cluster, instance_name ) ( vm, vm_fetched_by_passport ) = fulcrm_get_thing( uuid = instance_data[ 'uuid' ], source = ganeti_cluster, source_str = instance_data[ 'name' ], name = instance_data[ 'name' ], ) or ( {}, False ) if 'url' in vm: vm_url = vm[ 'url' ] else: vm_url = "/api/v2/thing/" vm = { "name": instance_name, "d": {}, } print( "NO vm for", instance_name ) vm[ 'source' ] = ganeti_cluster vm[ 'uuid' ] = instance_data[ 'uuid' ] vm[ 'source_str' ] = instance_data[ 'name' ] if 'fulcrm_collections' in vm[ 'd' ]: if "ganeti_instance" not in vm[ 'd' ].get( 'fulcrm_collections', [] ): vm[ 'd' ][ 'fulcrm_collections' ].append( 'ganeti_instance' ) else: vm[ 'd' ][ 'fulcrm_collections' ] = [ 'ganeti_instance' ] if "host" not in vm[ 'd' ][ 'fulcrm_collections' ]: vm[ 'd' ][ 'fulcrm_collections' ].append( 'host' ) vm[ 'created' ] = unix_to_8601( instance_data[ 'ctime' ] ) vm[ 'modified' ] = unix_to_8601( instance_data[ 'mtime' ] ) logs = [] logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_nics', [ { 'mac': instance_data[ 'nic.macs' ][ i ], 'uuid': instance_data[ 'nic.uuids' ][ i ], 'ip': instance_data[ 'nic.ips' ][ i ], 'link': instance_data[ 'custom_nicparams' ][ i ].get( 'link', None ), 'mode': instance_data[ 'custom_nicparams' ][ i ].get( 'mode', None ), 'vlan': ganeti_vlan( instance_data[ 'custom_nicparams' ][ i ].get( 'vlan', None ) ), } for i in range( len( instance_data[ 'nic.macs' ] ) ) ] ) ) logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_disks', [ { 'spindles': instance_data[ 'disk.spindles' ][ i ], 'uuid': instance_data[ 'disk.uuids' ][ i ], 'size': instance_data[ 'disk.sizes' ][ i ], 'template': instance_data[ 'disk_template' ], } for i in range( len( instance_data[ 'disk.spindles' ] ) ) ] ) ) ganeti_hvparams = instance_data[ 'hvparams' ] ganeti_hvparams.update( instance_data[ 'custom_hvparams' ] ) ganeti_beparams = instance_data[ 'beparams' ] ganeti_beparams.update( instance_data[ 'custom_beparams' ] ) logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_hvparams', ganeti_hvparams ) ) logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_beparams', ganeti_beparams ) ) logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_nodes', { 'primary_node': instance_data[ 'pnode' ], 'secondary_nodes': instance_data[ 'snodes' ], } ) ) logs.extend( fulcrm_set_d( vm[ 'd' ], 'name', instance_name ) ) addrs = set( vm[ 'd' ].get( 'addrs', [] ) ) try: for rr in resolver.resolve( instance_name, 'A' ): addrs.add( str( rr ) ) except dns.resolver.NXDOMAIN: pass except dns.resolver.NoAnswer: pass try: for rr in resolver.resolve( instance_name, 'AAAA' ): addrs.add( str( rr ) ) except dns.resolver.NXDOMAIN: pass except dns.resolver.NoAnswer: pass if addrs: vm[ 'd' ][ 'addr' ] = list( addrs ) vm[ 'd' ][ 'addr' ].sort() vm_is_new = vm.get( 'url', None ) is None if vm_is_new: vm = fulcrmapi.post( vm_url, vm ) else: vm = fulcrmapi.patch( vm_url, vm ) if vm_is_new or not vm_fetched_by_passport: passport = fulcrmapi.post( '/api/v2/passport/', { 'content_object': vm[ 'url' ], 'system': "gan2ful." + ganeti_cluster, 'passport': instance_data[ 'uuid' ], }, no_d = True ) if vm_is_new: vm, vm_fetched_by_passport = fulcrm_get_thing( uuid = instance_data[ 'uuid' ] ) fulcrmapi.post( '/api/v2/flag/', { 'content_object': vm[ 'url' ], 'flag': u"new instance created in ganeti: no corresponding thing record exists", 'severity': "warning", 'for_user': USER_MAREK, 'type': 'todo', }, no_d = True ) for log in logs: fulcrmapi.post( '/api/v2/l/', { 'content_object': vm[ 'url' ], 'log': log, 'system': 'gan2ful', }, no_d = True )