write urlqueryparse submodule for funky URL transforms

fulcrmpy/apiv2.py

@@ -3,6 +3,8 @@ import requests.exceptions
 import json
 import time
 
+from . import urlqueryparse
+
 class ApiKeyAuth( requests.auth.AuthBase ):
     def __init__( self, api_username, api_key ):
         self.api_username = api_username
@@ -22,28 +24,35 @@ class APIv2( object ):
         self.base = base
         self.auth = ApiKeyAuth( self.api_username, self.api_key )
 
-    def normalize_url( self, url, no_d = False, count = None ):
+    def normalize_path( self, path ):
-        if not url.startswith( self.base ):
+        if not path.startswith( self.api_path ):
-            if not url.startswith( self.api_path ):
+            if not path.startswith( "/" ):
-                if not url.startswith( "/" ):
+                path = "/" + path
-                    url = "/" + url
+            path = self.api_path + path
-                url = self.api_path + url
+        return path
-            url = self.base + url
 
-        if not no_d:
+    def normalize_url( self, url, no_d = False, query = None ):
-            if "?" in url:
+        url = urlqueryparse.set_url_components( url,
-                url = url + "&expand=d&count=" + str( count )
+                                                scheme = 'https',
+                                                netloc = 'fulcrm.org',
+                                                path = self.normalize_path )
+        new_query = urlqueryparse.get_query_fields( url )
+        if no_d:
+            if 'expand' in new_query:
+                new_query[ 'expand' ] = [ x for x in new_query[ 'expand' ] if not ( ( x == 'd' ) or ( x.endswith( '.d' ) ) ) ]
+        else:
+            if 'expand' in new_query:
+                new_query[ 'expand' ].append( 'd' )
             else:
-                url = url + "?expand=d&count=" + str( count )
+                new_query[ 'expand' ] = [ 'd' ]
 
-        if count is not None:
+        if query:
-            if "?" in url:
+            new_query.update( query )
-                url = url + "&count=" + str( count )
-            else:
-                url = url + "?count=" + str( count )
 
+        if 'expand' in new_query:
+            new_query[ 'expand' ] = [ ",".join( new_query[ 'expand'] ) ]
 
-        return url
+        return urlqueryparse.set_query_fields( url, new_query )
 
     def make_request( self, url, data, method, _countdown = 5 ):
         url = self.normalize_url( url )
@@ -78,7 +87,7 @@ class APIv2( object ):
 
     def get_many( self, url, count = 10, no_d = False ):
         next = None
-        url = self.normalize_url( url, no_d = no_d, count = count )
+        url = self.normalize_url( url, no_d = no_d, query = { 'count': [ count ] } )
 
         while True:
             result = self.make_request( url, None, 'GET' )

fulcrmpy/urlqueryparse.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+
+# based on myurlparse.py by alexwlchan
+# taken from: https://gist.github.com/alexwlchan/1956efe1acb1f2947cbd575651a3d529
+# see also: https://alexwlchan.net/2016/08/dealing-with-query-strings/
+
+try:  # Python 3+
+    from urllib.parse import ( parse_qs, parse_qsl, urlencode, urlparse, urlunparse )
+except ImportError:  # Python 2
+    from urllib import urlencode
+    from urlparse import parse_qs, parse_qsl, urlparse, urlunparse
+
+def get_query_fields( url ):
+    return dict( parse_qs( urlparse( url ).query ) )
+
+def set_query_fields( url, querydict ):
+    components = urlparse( url )
+    parseddict = dict( parse_qsl( urlparse( url ).query ) )
+    parseddict.update( querydict )
+    components = ( components.scheme,
+                   components.netloc,
+                   components.path,
+                   components.params,
+                   urlencode( list( parseddict.items() ) ),
+                   components.fragment
+                   )
+    return urlunparse( components )
+
+def set_url_components( url, scheme = None, netloc = None, path = None, params = None, query = None, fragment = None ):
+    components = urlparse( url )
+    components = ( callable( scheme ) and scheme( components.scheme ) or scheme or components.scheme,
+                   callable( netloc ) and netloc( components.netloc ) or netloc or components.netloc,
+                   callable( path ) and path( components.path ) or path or components.path,
+                   callable( params ) and params( components.params ) or params or components.params,
+                   callable( query ) and query( components.query ) or query or components.query,
+                   callable( fragment ) and fragment( components.fragment ) or fragment or components.fragment,
+                   )
+    return urlunparse( components )