commit 804417481a2b0153aed613f3a6a1bcf553399542 Author: Marek Isalski Date: Sun Jul 25 13:12:55 2021 +0100 initial import diff --git a/.hgignore b/.hgignore new file mode 100644 index 0000000..77801ed --- /dev/null +++ b/.hgignore @@ -0,0 +1,3 @@ +^\.venv/ +^Pipfile$ +^Pipfile\.lock$ diff --git a/g2f-sync b/g2f-sync new file mode 100755 index 0000000..145c30d --- /dev/null +++ b/g2f-sync @@ -0,0 +1,201 @@ +#!/usr/bin/python + +from fulcrmpy import apiv2 as fulcrm +import sys +import requests +from requests.auth import HTTPBasicAuth +import datetime +import dns.resolver +import json +import re + +resolver = dns.resolver.Resolver() +vlan_re = re.compile( r"([:\.][0-9]+)" ) + +THINGTYPE_VM = "/api/v2/thingtype/4/" +USER_MAREK = "/api/v2/user/1/" + +ganeti_cluster = sys.argv[ 1 ] +fulcrmapi = fulcrm.APIv2( sys.argv[ 2 ], sys.argv[ 3 ] ) +ganeti_auth = HTTPBasicAuth( sys.argv[ 4 ], sys.argv[ 5 ] ) + +def ganeti_get_instances( cluster ): + global ganeti_auth + cluster = "127.0.0.1" + req = requests.get( "https://" + cluster + ":5080/2/instances", verify = False, auth = ganeti_auth ) + return req.json() + +def ganeti_get_instance( cluster, name ): + global ganeti_auth + cluster = "127.0.0.1" + req = requests.get( "https://" + cluster + ":5080/2/instances/" + name, verify = False, auth = ganeti_auth ) + return req.json() + +def fulcrm_get_thing( uuid = None, source_str = None, source = None, name = None ): + if uuid: + for thing in fulcrmapi.get_many( "/api/v2/thing/", + query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], + "passport__system": [ "gan2ful.%s" % ganeti_cluster ], + "passport__passport": [ uuid ], + } ): + return ( thing, True ) + for thing in fulcrmapi.get_many( "/api/v2/thing/", + query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], + "uuid": [ uuid ], + } ): + return ( thing, False ) + if source_str: + if source: + for thing in fulcrmapi.get_many( "/api/v2/thing/", + query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], + "source": [ source ], + "source_str": [ source_str ], + } ): + return ( thing, False ) + else: + for thing in fulcrmapi.get_many( "/api/v2/thing/", + query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], + "source_str": [ source_str ], + } ): + return ( thing, False ) + if name: + for thing in fulcrmapi.get_many( "/api/v2/thing/", + query = { "expand": [ "d", "related_from_thing2things", "related_to_thing2things", "thing2thingtypes", "thing2thingtypes.thingtype" ], + "name": [ name ], + } ): + if thing[ 'source' ]: + print( '"%s" already exists source "%s" source_str "%s"' % ( name, thing[ 'source' ], thing[ 'source_str' ] ) ) + else: + return ( thing, False ) + +def ganeti_vlan( val ): + if val: + return [ x for ( i, x ) in enumerate( vlan_re.split( val ) ) if i % 2 ] + return None + +def unix_to_8601( t ): + if t: + return datetime.datetime.utcfromtimestamp( t ).strftime( "%Y-%m-%dT%H:%M:%SZ" ) + +def fulcrm_set_d( d, key, value ): + logs = [] + + if key in d: + if d[ key ] != value: + logs.append( "changing %s from %s" % ( repr( key ), json.dumps( value ) ) ) + d[ key ] = value + else: + d[ key ] = value + logs.append( "adding value for %s" % ( repr( key ), ) ) + return logs + + + +if __name__ == '__main__': + fulcrm_thing_cluster = fulcrm_get_thing( source = "ganeti-cluster", source_str = ganeti_cluster, name = ganeti_cluster ) + if not fulcrm_thing_cluster: + raise ValueError( "cannot find %s cluster in fulcrm" % ganeti_cluster ) + + for instance in ganeti_get_instances( ganeti_cluster ): + instance_name = instance[ 'id' ] + instance_data = ganeti_get_instance( ganeti_cluster, instance_name ) + + ( vm, vm_fetched_by_passport ) = fulcrm_get_thing( uuid = instance_data[ 'uuid' ], + source = ganeti_cluster, source_str = instance_data[ 'name' ], + name = instance_data[ 'name' ], + ) or ( {}, False ) + + if 'url' in vm: + vm_url = vm[ 'url' ] + else: + vm_url = "/api/v2/thing/" + vm = { "name": instance_name, + "d": {}, + } + print( "NO vm for", instance_name ) + + vm[ 'source' ] = ganeti_cluster + vm[ 'uuid' ] = instance_data[ 'uuid' ] + vm[ 'source_str' ] = instance_data[ 'name' ] + if 'fulcrm_collections' in vm[ 'd' ]: + if "ganeti_instance" not in vm[ 'd' ].get( 'fulcrm_collections', [] ): + vm[ 'd' ][ 'fulcrm_collections' ].append( 'ganeti_instance' ) + else: + vm[ 'd' ][ 'fulcrm_collections' ] = [ 'ganeti_instance' ] + if "host" not in vm[ 'd' ][ 'fulcrm_collections' ]: + vm[ 'd' ][ 'fulcrm_collections' ].append( 'host' ) + + vm[ 'created' ] = unix_to_8601( instance_data[ 'ctime' ] ) + vm[ 'modified' ] = unix_to_8601( instance_data[ 'mtime' ] ) + logs = [] + logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_nics', [ { 'mac': instance_data[ 'nic.macs' ][ i ], + 'uuid': instance_data[ 'nic.uuids' ][ i ], + 'ip': instance_data[ 'nic.ips' ][ i ], + 'link': instance_data[ 'custom_nicparams' ][ i ].get( 'link', None ), + 'mode': instance_data[ 'custom_nicparams' ][ i ].get( 'mode', None ), + 'vlan': ganeti_vlan( instance_data[ 'custom_nicparams' ][ i ].get( 'vlan', None ) ), + } for i in range( len( instance_data[ 'nic.macs' ] ) ) ] ) ) + logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_disks', [ { 'spindles': instance_data[ 'disk.spindles' ][ i ], + 'uuid': instance_data[ 'disk.uuids' ][ i ], + 'size': instance_data[ 'disk.sizes' ][ i ], + 'template': instance_data[ 'disk_template' ], + } for i in range( len( instance_data[ 'disk.spindles' ] ) ) ] ) ) + ganeti_hvparams = instance_data[ 'hvparams' ] + ganeti_hvparams.update( instance_data[ 'custom_hvparams' ] ) + ganeti_beparams = instance_data[ 'beparams' ] + ganeti_beparams.update( instance_data[ 'custom_beparams' ] ) + logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_hvparams', ganeti_hvparams ) ) + logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_beparams', ganeti_beparams ) ) + + logs.extend( fulcrm_set_d( vm[ 'd' ], 'ganeti_nodes', { 'primary_node': instance_data[ 'pnode' ], + 'secondary_nodes': instance_data[ 'snodes' ], + } ) ) + logs.extend( fulcrm_set_d( vm[ 'd' ], 'name', instance_name ) ) + + addrs = set( vm[ 'd' ].get( 'addrs', [] ) ) + try: + for rr in resolver.resolve( instance_name, 'A' ): + addrs.add( str( rr ) ) + except dns.resolver.NXDOMAIN: + pass + except dns.resolver.NoAnswer: + pass + try: + for rr in resolver.resolve( instance_name, 'AAAA' ): + addrs.add( str( rr ) ) + except dns.resolver.NXDOMAIN: + pass + except dns.resolver.NoAnswer: + pass + if addrs: + vm[ 'd' ][ 'addr' ] = list( addrs ) + vm[ 'd' ][ 'addr' ].sort() + + vm_is_new = vm.get( 'url', None ) is None + + if vm_is_new: + vm = fulcrmapi.post( vm_url, vm ) + else: + vm = fulcrmapi.patch( vm_url, vm ) + if vm_is_new or not vm_fetched_by_passport: + passport = fulcrmapi.post( '/api/v2/passport/', + { 'content_object': vm[ 'url' ], + 'system': "gan2ful." + ganeti_cluster, + 'passport': instance_data[ 'uuid' ], + }, no_d = True ) + + if vm_is_new: + vm, vm_fetched_by_passport = fulcrm_get_thing( uuid = instance_data[ 'uuid' ] ) + fulcrmapi.post( '/api/v2/flag/', + { 'content_object': vm[ 'url' ], + 'flag': u"new instance created in ganeti: no corresponding thing record exists", + 'severity': "warning", + 'for_user': USER_MAREK, + 'type': 'todo', + }, no_d = True ) + for log in logs: + fulcrmapi.post( '/api/v2/l/', + { 'content_object': vm[ 'url' ], + 'log': log, + 'system': 'gan2ful', + }, no_d = True ) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fa9e2cc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +git+https://gitea.faelix.net/FAELIX/fulcrmpy.git#egg=fulcrmpy +dnspython