RC 0.2

.gitignore

@@ -0,0 +1,6 @@
+settings.py
+*__pycache__/*
+*.pyc
+build/*
+dist/*
+*egg-info/*

AUTHORS

@@ -0,0 +1 @@
+Giuseppe De Marco <[email protected]>

LICENSE

@@ -0,0 +1,18 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+<pyMultiLDAP data connector and aggregator>
+           Copyright (C) 2019  Giuseppe De Marco
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

README.md

@@ -0,0 +1,244 @@
+pyMultiLDAP
+-----
+
+pyMultiLDAP can gather data from multiple LDAP servers, can do data aggregation and manipulation with rewrite rules.
+pyMultiLDAP can act also as a proxy server, behind openldap's slapd-sock backend or any custom implementation.
+
+### Features
+
+- LDAP client to many servers as a single one;
+- Custom functions to manipulate returning data (rewrite rules);
+- Export data in python dictionary, json or ldiff format;
+- Proxy Server, exposing a server daemon usable with [slapd-sock backend](https://www.openldap.org/software/man.cgi?query=slapd-sock).
+
+pyMultiLDAP do not write data to LDAP servers, it just permit us to handle readonly data
+in a way that could be very simple to automate smart data processing on-the-fly.
+
+See `example/settings.py.example` and `multildap/attr_rewrite.py` to understand how to configure and extend it.
+
+### Tested on
+
+- Debian9;
+- Debian10.
+
+### Setup
+Configure multiple connections and search paramenters in `settings.py`.
+
+Install
+````
+git clone https://github.com/peppelinux/pyMultiLDAP.git
+cd pyMultiLDAP
+pip install -r requirements
+python3 setup.py install
+````
+
+or use pipy [WIP]
+
+````
+pip install pyMultiLDAP
+````
+
+#### LdapClient Class usage
+````
+from multildap.client import LdapClient
+from settings import LDAP_CONNECTIONS
+
+lc = LdapClient(LDAP_CONNECTIONS['SAMVICE'])
+
+# get all the results
+lc.get()
+
+# apply a filter
+lc.get(search="(&(sn=de marco)(schacPersonalUniqueId=*DMRGPP83*))")
+````
+
+##### Search and get
+
+See `examples/run_test.py`.
+
+Difference between `.search` and `.get`:
+- *search* relyies on connection configuration and returns result as it come (raw);
+- *get* handles custom search filter and retrieve result as dictionary or json or ldif format. It also apply rewrite rules.
+
+````
+import copy
+
+from multildap.client import LdapClient
+from settings import LDAP_CONNECTIONS
+
+lc = LdapClient(LDAP_CONNECTIONS['DEFAULT'])
+
+kwargs = copy.copy(lc.conf)
+kwargs['search']['search_filter'] = "(&(sn=de medici)(givenName=aurora))"
+r = lc.search(**kwargs['search'])
+````
+
+#### Results in json format
+````
+from multildap.client import LdapClient
+from . settings import LDAP_CONNECTIONS
+
+
+for i in LDAP_CONNECTIONS:
+    lc = LdapClient(LDAP_CONNECTIONS[i])
+    print('# Results from: {} ...'.format(lc))
+
+    # get all as defined search_filter configured in settings connection
+    # but in json format
+    r = lc.get(format='json')
+    print(r)
+
+    # set a custom search as method argument
+    r = lc.get(search="(&(sn=de marco)(schacPersonalUniqueId=*DMRGPP345tg86H))", format='json')
+    print(r)
+
+    print('# End {}'.format(i))
+````
+
+#### Run the server
+
+Network address
+````
+multildapd.py -conf settings.py -port 1234
+````
+
+Unix domain socket (for slapd-sock backend)
+````
+multildapd.py -conf ./settings.py -loglevel "DEBUG" -socket /var/run/multildap.sock -pid /var/run/multildap.pid -uid openldap
+````
+
+Dummy test without any ldap client connection configured, just to test slapd-sock:
+````
+multildapd.py -conf ./settings.py -dummy -loglevel "DEBUG" -socket /var/run/multildap.sock -pid /var/run/multildap.pid
+````
+
+Test Unix domain socket from cli
+````
+nc -U /tmp/multildap.sock
+````
+
+#### Interfacing it with OpenLDAP slapd-sock
+
+The  [Slapd-sock](https://www.openldap.org/software/man.cgi?query=slapd-sock)
+ backend  to  slapd  uses  an external program to handle
+ queries. This makes it
+ possible to have a pool of processes, which persist  between  requests.
+ This  allows  multithreaded operation and a higher level of efficiency.
+ Multildapd  listens  on  a  Unix  domain  socket and it must have  been  started  independently;
+
+This  module  may  also  be  used  as  an  overlay on top of some other
+ database.  Use as an overlay allows external actions to be triggered in
+ response to operations on the main database.
+
+#### Configure slapd-sock as database
+
+Add the module.
+````
+ldapadd -Y EXTERNAL -H ldapi:/// <<EOF
+dn: cn=module,cn=config
+objectClass: olcModuleList
+cn: module
+olcModuleLoad: back_sock.la
+EOF
+````
+
+Create the database.
+````
+ldapadd -Y EXTERNAL -H ldapi:/// <<EOF
+dn: olcDatabase={4}sock,cn=config
+objectClass: olcDbSocketConfig
+olcDatabase: {4}sock
+olcDbSocketPath: /var/run/multildap.sock
+olcSuffix: dc=proxy,dc=testunical,dc=it
+olcDbSocketExtensions: binddn peername ssf
+EOF
+````
+
+Add an Overlay if you want to wrap an existing backend
+````
+ldapmodify -H ldapi:// -Y EXTERNAL <<EOF
+dn: olcOverlay=sock,olcDatabase={1}mdb,cn=config
+changetype: add
+objectClass: olcConfig
+objectClass: olcOverlayConfig
+objectClass: olcOvSocketConfig
+olcOverlay: sock
+olcDbSocketPath: /var/run/multildap/multildap.sock
+olcOvSocketOps: bind unbind search
+olcOvSocketResps: search
+EOF
+````
+
+Remember to configure an ACL otherwise only `ldapsearch -H ldapi:// -Y EXTERNAL` as root would fetch ldif.
+Remember to add a space char `' '` after every olaAccess line, otherwise you'll get `Implementation specific error(80)`.
+
+````
+export BASEDC="dc=testunical,dc=it"
+
+ldapadd -Y EXTERNAL -H ldapi:/// <<EOF
+dn: olcDatabase={4}sock,cn=config
+changeType: modify
+replace: olcAccess
+olcAccess: to * 
+ by dn.exact=gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth manage 
+ by * break
+# the following permits self BIND by users
+olcAccess: to dn.subtree="dc=proxy,$BASEDC" 
+ by self read 
+ by * break
+# the following two permits SEARCH by idp and foreign auth system
+olcAccess: to dn.subtree="ou=people,$BASEDC"  
+ by dn.children="ou=auth,$BASEDC" read 
+ by self read 
+ by * break 
+olcAccess: to dn.subtree="ou=people,$BASEDC" 
+ by dn.children="ou=idp,$BASEDC" read 
+ by self read 
+ by * break 
+olcAccess: to * 
+ by anonymous auth 
+ by * break 
+EOF
+````
+
+Authentication  (BIND) on top of the multildapd must be configured with attribute
+`rewrite_dn_to` regarding every connections in the settings.py. If abstent the specified connection will be excluded from authentication.
+TODO: _adopt openldap proxy authz statements_.
+
+````
+ldapsearch -H ldap://localhost:389 -D "uid=peppe,dc=proxy,dc=testunical,dc=it" -w thatsecret -b 'uid=peppe,dc=proxy,dc=unical,dc=it'
+````
+
+#### Hints
+
+See databases currently installed:
+- `ldapsearch -Y EXTERNAL -H ldapi:/// -b 'cn=config' -LLL  "olcDatabase=*"`;
+- Use `client_strategy = RESTARTABLE` instead of `REUSABLE` in your settings.py for better performances;
+- A Backend can not be deleted via ldapdelete/modify until OpenLDAP 2.5 will be released;
+- Changing the socket path
+````
+ldapmodify -Y EXTERNAL -H ldapi:/// <<EOF
+dn: olcDatabase={4}sock,cn=config
+changetype: modify
+replace: olcDbSocketPath
+olcDbSocketPath: /var/run/multildap.sock
+EOF
+````
+- Deploy a dummy socket listener with socat, just to debug incoming connection from slapd-sock.
+
+````
+socat -s UNIX-LISTEN:/tmp/slapd-sock,umask=000,fork EXEC:"$your_command"
+````
+
+#### Other slapd-sock resources:
+
+- [slapsock](https://build.opensuse.org/package/show/home:stroeder:AE-DIR/python-slapdsock)
+- [slapd-trigger](https://github.com/jclain/slapd-trigger)
+- [ldap.h search scopes](https://github.com/openldap/openldap/blob/master/include/ldap.h#L581)
+- [slapd-sock in OpenLDAP ML](https://www.openldap.org/cgi-bin/wilma_glimpse/openldap-technical?query=slapd-sock&Search=Search&errors=0&maxfiles=50&maxlines=10&.cgifields=lineonly&.cgifields=restricttofiles&.cgifields=filelist&.cgifields=partial&.cgifields=case)
+
+
+#### Todo
+
+- Example configuration with slapd's Proxy Authorization Rules (authzTo: dn.regex:^uid=[^,]*,dc=example,dc=com$);
+- Only SEARCH, BIND and UNBIND is usable, other LDAP methods should be implemented;

examples/README.asyncio.md

@@ -0,0 +1,2 @@
+- https://github.com/hughrobb/aioldap3
+- https://github.com/cannatag/ldap3/issues/182

examples/ldap_aio.py

@@ -0,0 +1,41 @@
+# https://www.pythonsheets.com/notes/python-asyncio.html
+import asyncio
+
+from client import LdapClient
+from settings import LDAP_CONNECTIONS
+
+result_set = []
+LDAP_SERVERS = []
+
+async def get_result(lc):
+    await asyncio.sleep(0.001)
+    return lc.get(format='dict')
+
+async def get_server(CONF):
+    return LdapClient(CONF)
+
+async def ensure_connection(lc):
+    await asyncio.sleep(0.001)
+    lc.ensure_connection()
+
+async def connect(lc_id: int):
+    CONF = list(LDAP_CONNECTIONS.keys())[lc_id]
+    lc = await get_server(LDAP_CONNECTIONS[CONF])
+    # LDAP_SERVERS.append(lc)
+    print('Connectign and gathering {} [{}]'.format(lc, CONF))
+    # await ensure_connection(lc)
+    try:
+        result = await get_result(lc)
+        result_set.extend(result)
+        print('Get {} results from {}'.format(len(result), lc))
+    except Exception as e:
+        print('-- Fail to connect to {} [{}]'.format(lc, CONF))
+        print(e)
+
+async def main():
+    await asyncio.gather(*(connect(n) for n in range(len(LDAP_CONNECTIONS.keys()))))
+    print('Done')
+
+if __name__ == '__main__':
+    asyncio.run(main())
+    print(len(result_set))

examples/ldap_gevent.py

@@ -0,0 +1,13 @@
+import gevent
+from gevent import monkey; monkey.patch_all()
+
+from client import LdapClient
+from settings import LDAP_CONNECTIONS
+
+result_set = []
+LDAP_SERVERS = [LdapClient(conf) for conf in LDAP_CONNECTIONS.values()]
+
+jobs = [gevent.spawn(conn.get) for conn in LDAP_SERVERS]
+gevent.joinall(jobs, timeout=2)
+for job in jobs:
+    print(job.value)

examples/ldap_gevent_server_echo.py

@@ -0,0 +1,28 @@
+#import gevent
+
+#from gevent import monkey; monkey.patch_all()
+from gevent.server import StreamServer
+
+from client import LdapClient
+from settings import LDAP_CONNECTIONS
+
+
+def handle(socket, address):
+    print('new connection from {}'.format(address))
+    rfileobj = socket.makefile(mode='rb')
+    while True:
+        line = rfileobj.readline()
+        if not line:
+            print("client {} disconnected".format(address))
+            break
+        elif line.strip().lower() == b'quit':
+            print("client {} quit".format(address))
+            break
+        elif line == b'\n': continue
+        else:
+            socket.sendall(b'recv: '+line)
+        print("< {}".format(line))
+    rfileobj.close()
+
+server = StreamServer(('127.0.0.1', 1234), handle) # creates a new server
+server.serve_forever() # start accepting new connections

examples/ldaptor/ldap-merger.py

@@ -0,0 +1,17 @@
+#! /usr/bin/env python
+
+from twisted.application import service, internet
+from twisted.internet import protocol
+from ldaptor.config import LDAPConfig
+from ldaptor.protocols.ldap.merger import MergedLDAPServer
+
+application = service.Application("LDAP Merger")
+
+configs = [LDAPConfig(serviceLocationOverrides={"": ('host1.unical.it', 389)}),
+           LDAPConfig(serviceLocationOverrides={"": ('host2.unical.it', 636)})
+           ]
+use_tls = [True, True]
+factory = protocol.ServerFactory()
+factory.protocol = lambda: MergedLDAPServer(configs, use_tls)
+mergeService = internet.TCPServer(3899, factory)
+mergeService.setServiceParent(application)

examples/ldaptor/requirements

@@ -0,0 +1,5 @@
+ldaptor
+passlib
+pyparsing
+twisted[tls]
+zope.interface