Add higher level LDAP cache API ldapcache is an API which makes ldapwatch easier to use. Basically, it maintains a cache of the contents of the directory and invokes registered callbacks when any changes are made e.g.: def handle_change(cache, type, dn, prev_dn, attrs, prev_attrs): print "%s changed" % dn cache = LDAPCache("cache.ldif", "ldap://localhost/", "dc=foo,dc=org") cache.add_monitor(handle_change) while True: cache.run() cache.remove_monitor(handle_change) Note, that ldapcache also saves its cache to disk so that when it starts up it can compare the contents of the directory to the on-disk cache and detect any changes made while it wasn't running. Signed-off-by: Mark McLoughlin diff -r b260d1935f73 ipa-server/ipa-configd/ipaconfigd/Makefile.am --- a/ipa-server/ipa-configd/ipaconfigd/Makefile.am Fri Jan 25 17:45:01 2008 +0000 +++ b/ipa-server/ipa-configd/ipaconfigd/Makefile.am Fri Jan 25 17:45:12 2008 +0000 @@ -11,5 +11,7 @@ ipaconfigddir = $(pythondir)/ipaconf ipaconfigddir = $(pythondir)/ipaconfigd/ ipaconfigd_PYTHON = \ __init__.py \ + config.py \ + ldapcache.py \ main.py \ $(NULL) diff -r b260d1935f73 ipa-server/ipa-configd/ipaconfigd/config.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ipa-server/ipa-configd/ipaconfigd/config.py Fri Jan 25 17:45:12 2008 +0000 @@ -0,0 +1,22 @@ +# +# Copyright (C) 2008 Red Hat +# see file 'COPYING' for use and warranty information +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 only +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +# Number of milliseconds to wait before attempting to reconnect to directory +LDAPCACHE_N_MS_BEFORE_RECONNECT = 100 +# Number of time to try re-connecting - wait period will double each time +LDAPCACHE_N_RECONNECTS = 10 diff -r b260d1935f73 ipa-server/ipa-configd/ipaconfigd/ldapcache.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ipa-server/ipa-configd/ipaconfigd/ldapcache.py Fri Jan 25 17:45:12 2008 +0000 @@ -0,0 +1,308 @@ +# +# Copyright (C) 2008 Red Hat +# see file 'COPYING' for use and warranty information +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; version 2 only +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +import os +import errno +import time +import ldap +import ldif +import ldapwatch +from config import * + +TYPE_INVALID = ldapwatch.TYPE_INVALID +TYPE_ADD = ldapwatch.TYPE_ADD +TYPE_DELETE = ldapwatch.TYPE_DELETE +TYPE_MODIFY = ldapwatch.TYPE_MODIFY +TYPE_MODDN = ldapwatch.TYPE_MODDN + +def dprint(fmt, *args): + print fmt % args + pass + +def type_to_str(type): + if type == TYPE_INVALID: + return "invalid" + elif type == TYPE_ADD: + return "add" + elif type == TYPE_DELETE: + return "delete" + elif type == TYPE_MODIFY: + return "modify" + elif type == TYPE_MODDN: + return "moddn" + else: + return "unknown" + +def write_ldif_file(path, entries): + """Write an LDIF file to @path containing @entries. + + Care is taken to ensure that the existing file is only + overwritten if the new file can be successfully written. + """ + temp_path = path + ".new" + try: + f = file(temp_path, "w") + except: + temp_path = None + f = file(path, "w") + + try: + ldif_writer = ldif.LDIFWriter(f) + for dn in entries.keys(): + ldif_writer.unparse(dn, entries[dn]) + f.close() + except: + if not temp_path is None: + os.remove(temp_path) + raise + + if not temp_path is None: + os.rename(temp_path, path) + +def read_ldif_file(path, must_exist = True): + """Read the contents of the LDIF file specified by @path + and return the contents as a dict of entries. + + If @must_exist is #False, then an empty dict is + returned if the file does not exist. + """ + try: + f = file(path, "r") + except IOError, e: + if must_exist or e.errno != errno.ENOENT: + raise + return {} + + ldif_parser = ldif.LDIFRecordList(f) + ldif_parser.parse() + + entries = {} + for (dn, attrs) in ldif_parser.all_records: + entries[dn] = attrs + + return entries + +class LDAPCache: + """LDAPCache is a class which allows you to monitor a portion + of a directory for additions, deletions, changes and moves. + + LDAPCache will also detect modifications made in between runs + by saving the cache to a file. + """ + + def __init__(self, cache_path, uri, base, bind_dn = None, bind_passwd = None): + """LDAPCache constructure: + + @cache_path: the path to the LDIF file used as an on-disk cache + @uri: the LDAP uri used to connect to the directory server. + @base: the portion of the directory you wish to monitor + @bind_dn: the DN to bind as (optional) + @bind_passwd: the passwd to use when binding as @bind_dn (optional) + """ + self._cache_path = cache_path + + self._uri = uri + self._base = base + self._bind_dn = bind_dn + self._bind_passwd = bind_passwd + + self._monitors = [] + + self._cnx = None + self._watch = None + self._entries = {} + + def add_monitor(self, monitor_func): + """Add a monitor callback which will be invoked when any + modifications to the directory are detect. + + @monitor_func should take five arguments: + - @cache: the LDAPCache + - @type: the type of change; TYPE_ADD, TYPE_DELETE, + TYPE_MODIFY and TYPE_MODDN are all possible + values + - @dn: the DN of the affected entry + - @prev_dn: the original DN of the entry (with TYPE_MODDN) + - @attrs: the current attributes of the affected entry + - @prev_attrs: the previous attributes of the affected entry + """ + self._monitors.append(monitor_func) + + def remove_monitor(self, monitor_func): + """Remove a monitor callback which was previously added + with add_monitor. + """ + self._monitors.remove(monitor_func) + + def __invoke_monitors(self, type, dn, prev_dn, attrs, old_attrs): + """Invoke all previously added monitor callbacks with the + supplied arguments. + """ + for monitor_func in self._monitors: + monitor_func(self, type, dn, prev_dn, attrs, old_attrs) + + def __write_cache(self): + """Write the cache of directory contents to disk. + """ + dprint("Writing cache of %d entries to '%s'", + len(self._entries), self._cache_path) + write_ldif_file(self._cache_path, self._entries) + + def __compare_with_cached(self): + """Read the on-disk cache of the directory contents and compare + with the current directory contents. If any modifications are + detected, invoke monitor callbacks as appropriate. + """ + dprint("Reading cache from '%s'", self._cache_path) + cached_entries = read_ldif_file(self._cache_path, False) + + for dn in self._entries.keys(): + if not dn in cached_entries: + self.__invoke_monitors(TYPE_ADD, dn, None, self._entries[dn], None) + elif not self._entries[dn] == cached_entries[dn]: + self.__invoke_monitors(TYPE_MODIFY, dn, None, self._entries[dn], cached_entries[dn]) + + for dn in cached_entries.keys(): + if not dn in self._entries: + self.__invoke_monitors(TYPE_DELETE, dn, None, None, cached_entries[dn]) + + self.__write_cache() + + def __connect(self): + """Connect to the directory server, bind if required, read + the contents of the directory and compare with the on-disk + cache. + """ + dprint("Connecting to directory at '%s'", self._uri) + + self._cnx = ldap.initialize(self._uri) + + if self._bind_dn and self._bind_passwd: + self._cnx.simple_bind_s(self._bind_dn, self._bind_passwd) + + self._watch = ldapwatch.LDAPWatch(self._uri, self._base, self._bind_dn, self._bind_passwd) + + dprint("Successfully connected; loading contents") + + self._entries = {} + for (dn, attrs) in self._cnx.search_s(self._base, ldap.SCOPE_SUBTREE): + self._entries[dn] = attrs + + self.__compare_with_cached() + + def __try_reconnect(self): + """Blocks for a configurable period, before return #True indicating + that the caller should try and re-connect to the directory. + + Returns #False if we've already tried to re-connect too many times. + """ + self._cnx = None + self._watch = None + self._entries = {} + + if self._n_reconnects < 1: + dprint("Giving up attempting to reconnect to directory"); + + return False + else: + dprint("Waiting %d milliseconds before attempting to reconnect to directory", + self._n_ms_before_reconnect) + + time.sleep(self._n_ms_before_reconnect / 1000.0) + self._n_ms_before_reconnect *= 2 + self._n_reconnects -= 1 + + return True + + def __process_pending_changes(self, timeout = None): + """Poll the directory for changes, blocking until some changes + have been received, and handle those changes by invoking + monitor functions as appropriate. + + @timeout specifies a floating point timeout in seconds which + controls the maximum length of time this iteration will + block for. If @timeout is unspecified or #None, the function + will block indefinitely until results are received. If @timeout + is 0.0, the function will return immediately if no results + are pending. + + Returns #True if any changes were processed, #False if the + timeout occurred before any changes were received. + """ + dprint("Blocking for changes from directory; timeout = '%s'", timeout) + + n_changes = 0 + for (dn, type, prev_dn) in self._watch.get_changes(timeout): + n_changes += 1 + + old_attrs = None + if dn in self._entries: + old_attrs = self._entries[dn] + + if prev_dn and prev_dn in self._entries: + del self._entries[prev_dn] + + try: + self._entries[dn] = self._cnx.search_s(dn, ldap.SCOPE_BASE)[0][1] + except ldap.NO_SUCH_OBJECT: + del self._entries[dn] + + attrs = None + if dn in self._entries: + attrs = self._entries[dn] + + self.__invoke_monitors(type, dn, prev_dn, attrs, old_attrs) + + self.__write_cache() + + dprint("Processed %d changes", n_changes) + + return n_changes > 0 + + def run(self, timeout = None): + """Monitor the directory server and invoke monitor callbacks + when any modifications are detected. + + @timeout specifies a floating point timeout in seconds which + controls the maximum length of time each iteration will + block for. If @timeout is unspecified or #None, the function + will block indefinitely. If @timeout is 0.0, the function will + return as soon as an iteration produces no results. + + Returns if the operation timed out. + """ + self._n_ms_before_reconnect = LDAPCACHE_N_MS_BEFORE_RECONNECT + self._n_reconnects = LDAPCACHE_N_RECONNECTS + while True: + try: + if self._cnx is None: + self.__connect() + elif not self.__process_pending_changes(timeout): + break + except ldapwatch.LDAPError: + if not self.__try_reconnect(): + raise + + def get_entries(self): + """Return the a dictioary of the currently cached entries. The + key of the dictionary is the entry's DN while the value of + the dictionary will be another dictionary containing the entry's + attributes. That dictionary is indexed by attribute name and + its values are lists of attribute values. + """ + return self._entries diff -r b260d1935f73 ipa-server/ipa-configd/ipaconfigd/main.py --- a/ipa-server/ipa-configd/ipaconfigd/main.py Fri Jan 25 17:45:01 2008 +0000 +++ b/ipa-server/ipa-configd/ipaconfigd/main.py Fri Jan 25 17:45:12 2008 +0000 @@ -25,7 +25,39 @@ import os import os import os.path import optparse +import ldapcache from config import * + +def dprint (fmt, *args): + print fmt % args + pass + +def debug_directory_change(cache, type, dn, prev_dn, attrs, prev_attrs): + def dprint_attrs(attrs): + if attrs is None: + _dprint (" -> ") + return + + for attr in attrs.keys(): + for value in attrs[attr]: + dprint(" -> %s: %s", attr, value) + + if type == ldapcache.TYPE_ADD: + dprint("'%s' added; attributes:", dn) + dprint_attrs(attrs) + elif type == ldapcache.TYPE_DELETE: + dprint("'%s' deleted; previous attributes:", dn) + dprint_attrs(prev_attrs) + elif type == ldapcache.TYPE_MODIFY: + dprint("'%s' modified; new attributes:", dn) + dprint_attrs (attrs) + dprint("old attributes:") + dprint_attrs(prev_attrs) + elif type == ldapcache.TYPE_MODDN: + dprint("'%s' moved to '%s'; attributes", dn, prev_dn) + dprint_attrs(attrs) + else: + dprint("Unknown directory change type '%d'", type) def daemonize(): """Detach from the controlling terminal and run in the background.""" @@ -92,9 +124,17 @@ def main (args): write_pid_file(options.pidfile) try: + cache = ldapcache.LDAPCache("/var/cache/ipa/configd-ldapcache.ldif", + "ldap://localhost/", "cn=config,dc=IPA") + + if options.debug: + cache.add_monitor(debug_directory_change) + while True: - import time - time.sleep(10) + cache.run() + + if options.debug: + cache.remove_monitor(debug_directory_change) finally: if options.pidfile and os.path.exists(options.pidfile): os.remove(options.pidfile) diff -r b260d1935f73 ipa-server/ipa-install/ipa-server-install --- a/ipa-server/ipa-install/ipa-server-install Fri Jan 25 17:45:01 2008 +0000 +++ b/ipa-server/ipa-install/ipa-server-install Fri Jan 25 17:45:12 2008 +0000 @@ -280,6 +280,8 @@ def uninstall(): ipaserver.dsinstance.DsInstance().uninstall() sysrestore.restore_file("/etc/hosts") sysrestore.restore_file("/etc/ipa/ipa.conf") + if os.path.exists("/var/cache/ipa/configd-ldapcache.ldif"): + os.remove("/var/cache/ipa/configd-ldapcache.ldif") return 0 def main():