commit 30738329e1b23f6ec0106172399b97ff100a31b0 Author: Stefan Hacker Date: Sat Nov 20 03:36:50 2010 +0100 Nearly finished basic mumo application. config, worker and mumo_module have test coverage. mumo_manager is not yet covered and most likely not right yet. diff --git a/config.py b/config.py new file mode 100644 index 0000000..6330c4e --- /dev/null +++ b/config.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import ConfigParser + +class Config(object): + """ + Small abstraction for config loading + """ + + def __init__(self, filename = None, default = None): + if not filename or not default: return + cfg = ConfigParser.ConfigParser() + cfg.optionxform = str + cfg.read(filename) + + for h,v in default.iteritems(): + if not v: + # Output this whole section as a list of raw key/value tuples + try: + self.__dict__[h] = cfg.items(h) + except ConfigParser.NoSectionError: + self.__dict__[h] = [] + else: + self.__dict__[h] = Config() + for name, val in v.iteritems(): + conv, vdefault = val + try: + self.__dict__[h].__dict__[name] = conv(cfg.get(h, name)) + except (ValueError, ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.__dict__[h].__dict__[name] = vdefault + +def x2bool(s): + """Helper function to convert strings from the config to bool""" + if isinstance(s, bool): + return s + elif isinstance(s, basestring): + return s.lower() in ['1', 'true'] + raise ValueError() diff --git a/config_test.py b/config_test.py new file mode 100644 index 0000000..725dc4c --- /dev/null +++ b/config_test.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from config import Config, x2bool +from tempfile import mkstemp +import os + +def create_file(content = None): + """ + Creates a temp file filled with 'content' and returns its path. + The file has to be manually deleted later on + """ + fd, path = mkstemp() + f = os.fdopen(fd, "wb") + if content: + f.write(content) + f.flush() + f.close() + return path + +class ConfigTest(unittest.TestCase): + cfg_content = """[world] +domination = True +somestr = Blabla +somenum = 10 +testfallbacknum = asdas +""" + + cfg_default = {'world':{'domination':(x2bool, False), + 'somestr':(str, "fail"), + 'somenum':(int, 0), + 'somenumtest':(int, 1)}, + 'somethingelse':{'bla':(str, "test")}} + + def setUp(self): + pass + + def tearDown(self): + pass + + + def testEmpty(self): + path = create_file() + try: + cfg = Config(path, self.cfg_default) + assert(cfg.world.domination == False) + assert(cfg.world.somestr == "fail") + assert(cfg.world.somenum == 0) + self.assertRaises(AttributeError, getattr, cfg.world, "testfallbacknum") + assert(cfg.somethingelse.bla == "test") + finally: + os.remove(path) + + def testX2bool(self): + assert(x2bool("true") == True) + assert(x2bool("false") == False) + assert(x2bool("TrUe") == True) + assert(x2bool("FaLsE") == False) + assert(x2bool("0") == False) + assert(x2bool("1") == True) + assert(x2bool("10") == False) + assert(x2bool("notabool") == False) + + def testConfig(self): + path = create_file(self.cfg_content) + try: + try: + cfg = Config(path, self.cfg_default) + except Exception, e: + print e + assert(cfg.world.domination == True) + assert(cfg.world.somestr == "Blabla") + assert(cfg.world.somenum == 10) + self.assertRaises(AttributeError, getattr, cfg.world, "testfallbacknum") + assert(cfg.somethingelse.bla == "test") + finally: + os.remove(path) + + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() \ No newline at end of file diff --git a/modules/test.py b/modules/test.py new file mode 100644 index 0000000..2864b72 --- /dev/null +++ b/modules/test.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from mumo_module import (x2bool, + MumoModule, + logModFu) + +class test(MumoModule): + default_config = {'testing':{'tvar': (int , 1), + 'novar': (str, 'no bernd')}} + + def __init__(self, cfg_file, manager): + MumoModule.__init__(self, "test", manager, cfg_file) + log = self.log() + cfg = self.cfg() + log.debug("tvar: %s", cfg.testing.tvar) + log.debug("novar: %s", cfg.testing.novar) + + @logModFu + def unload(self): + pass + + @logModFu + def connected(self): + manager = self.manager() + log = self.log() + log.debug("Ice connected, register for everything out there") + manager.enlistMetaCallbackHandler(self) + manager.enlistServerCallbackHandler(self, manager.SERVER_ALL_TRACK) + manager.enlistServerContextCallbackHandler(self, manager.SERVER_ALL_TRACK) + + @logModFu + def disconnected(self): + self.log().debug("Ice list") + # + #--- Meta callback functions + # + + @logModFu + def started(self, server, context = None): + pass + + @logModFu + def stopped(self, server, context = None): + pass + + # + #--- Server callback functions + # + @logModFu + def userConnected(self, state, context = None): + pass + + @logModFu + def userDisconnected(self, state, context = None): + pass + + @logModFu + def userStateChanged(self, state, context = None): + pass + + @logModFu + def channelCreated(self, state, context = None): + pass + + @logModFu + def channelRemoved(self, state, context = None): + pass + + @logModFu + def channelStateChanged(self, state, context = None): + pass + + # + #--- Server context callback functions + # + @logModFu + def contextAction(self, action, user, session, channelid, context = None): + pass \ No newline at end of file diff --git a/mumo_manager.py b/mumo_manager.py new file mode 100644 index 0000000..d67dea3 --- /dev/null +++ b/mumo_manager.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import Queue +from worker import Worker, local_thread, local_thread_blocking +import sys +import os + +class FailedLoadModuleException(Exception): + pass + +class FailedLoadModuleConfigException(FailedLoadModuleException): + pass + +class FailedLoadModuleImportException(FailedLoadModuleException): + pass + +class FailedLoadModuleInitializationException(FailedLoadModuleException): + pass + +def debug_log(fu, enable = True): + def new_fu(*args, **kwargs): + self = args[0] + log = self.log() + skwargs = [','.join(['%s=%s' % (karg,repr(arg)) for karg, arg in kwargs])] + sargs = [','.join([str(arg) for arg in args[1:]])] + '' if not skwargs else (',' + skwargs) + + call = "%s(%s)" % (fu.__name__, sargs) + log.debug() + res = fu(*args, **kwargs) + log.debug("%s -> %s", call, repr(res)) + return res + return new_fu if enable else fu + +debug_me = True + +class MumoManagerRemote(object): + """ + Manager object handed to MumoModules. This module + acts as a remote for the MumoModule with which it + can register/unregister to/from callbacks as well + as do other signaling to the master MumoManager. + """ + + SERVERS_ALL = [-1] ## Applies to all servers + + def __init__(self, master, name, queue): + self.__master = master + self.__name = name + self.__queue = queue + + def getQueue(self): + return self.__queue + + def subscribeMetaCallbacks(self, handler, servers = MumoManagerRemote.SERVERS_ALL): + """ + Subscribe to meta callbacks. Subscribes the given handler to the following + callbacks: + + >>> started(self, server, context = None) + >>> stopped(self, server, context = None) + + @param servers: List of server IDs for which to subscribe. To subscribe to all + servers pass SERVERS_ALL. + @param handler: Object on which to call the callback functions + """ + return self.__master.subscribeMetaCallbacks(self.__queue, handler, servers) + + def unsubscribeMetaCallbacks(self, handler, servers = MumoManagerRemote.SERVERS_ALL): + """ + Unsubscribe from meta callbacks. Unsubscribes the given handler from callbacks + for the given servers. + + @param servers: List of server IDs for which to unsubscribe. To unsubscribe from all + servers pass SERVERS_ALL. + @param handler: Subscribed handler + """ + return self.__master.unscubscribeMetaCallbacks(self.__queue, handler, servers) + + def subscribeServerCallbacks(self, handler, servers = MumoManagerRemote.SERVERS_ALL): + """ + Subscribe to server callbacks. Subscribes the given handler to the following + callbacks: + + >>> userConnected(self, state, context = None) + >>> userDisconnected(self, state, context = None) + >>> userStateChanged(self, state, context = None) + >>> channelCreated(self, state, context = None) + >>> channelRemoved(self, state, context = None) + >>> channelStateChanged(self, state, context = None) + + @param servers: List of server IDs for which to subscribe. To subscribe to all + servers pass SERVERS_ALL. + @param handler: Object on which to call the callback functions + """ + return self.__master.subscribeServerCallbacks(self.__queue, handler, servers) + + def unsubscribeServerCallbacks(self, handler, servers = MumoManagerRemote.SERVERS_ALL): + """ + Unsubscribe from server callbacks. Unsubscribes the given handler from callbacks + for the given servers. + + @param servers: List of server IDs for which to unsubscribe. To unsubscribe from all + servers pass SERVERS_ALL. + @param handler: Subscribed handler + """ + return self.__master.unsubscribeServerCallbacks(self.__queue, handler, servers) + + def subscribeContextCallbacks(self, handler, servers = MumoManagerRemote.SERVERS_ALL): + """ + Subscribe to context callbacks. Subscribes the given handler to the following + callbacks: + + >>> contextAction(self, action, user, session, channelid, context = None) + + @param servers: List of server IDs for which to subscribe. To subscribe to all + servers pass SERVERS_ALL. + @param handler: Object on which to call the callback functions + """ + return self.__master.subscribeContextCallbacks(self.__queue, handler, servers) + + def unsubscribeContextCallbacks(self, handler, servers = MumoManagerRemote.SERVERS_ALL): + """ + Unsubscribe from context callbacks. Unsubscribes the given handler from callbacks + for the given servers. + + @param servers: List of server IDs for which to unsubscribe. To unsubscribe from all + servers pass SERVERS_ALL. + @param handler: Subscribed handler + """ + return self.__master.unsubscribeContextCallbacks(self.__queue, handler, servers) + + + +class MumoManager(Worker): + MAGIC_ALL = -1 + + def __init__(self, cfg): + Worker.__init__(self, "MumoManager") + self.queues = {} # {queue:module} + self.modules = {} # {name:module} + self.imports = {} # {name:import} + self.cfg = cfg + + self.metaCallbacks = {} # {sid:{queue:[handler]}} + self.serverCallbacks = {} + self.contextCallbacks = {} + + def __add_to_dict(self, mdict, queue, handler, servers): + for server in servers: + if server in mdict: + if queue in mdict[server]: + if not handler in mdict[server][queue]: + mdict[server][queue].append(handler) + else: + mdict[server][queue] = [handler] + else: + mdict[server] = {queue:[handler]} + + def __rem_from_dict(self, mdict, queue, handler, servers): + for server in servers: + try: + mdict[server][queue].remove(handler) + except KeyError, ValueError: + pass + + def __announce_to_dict(self, mdict, servers, function, *args, **kwargs): + """ + Call function on handlers for specific servers in one of our handler + dictionaries. + + @param mdict Dictionary to announce to + @param servers: Servers to announce to, ALL is always implied + @param function: Function the handler should call + @param args: Arguments for the function + @param kwargs: Keyword arguments for the function + """ + # Announce to all handlers registered to all events + for queue, handlers in mdict[self.MAGIC_ALL].iteritems(): + for handler in handlers: + self.__call_remote(queue, handler, args, kwargs) + + # Announce to all handlers of the given serverlist + for server in servers: + for queue, handler in mdict[server].iteritems(): + self.__call_remote(queue, handler, args, kwargs) + + def __call_remote(self, queue, handler, *args, **kwargs): + queue.put((None, handler, args, kwargs)) + + # + #--- Module self management functionality + # + + @local_thread + def subscribeMetaCallbacks(self, queue, handler, servers): + """ + @param queue Target worker queue + @see MumoManagerRemote + """ + return self.__add_to_dict(self.metaCallbacks, queue, handler, servers) + + @local_thread + def unsubscribeMetaCallbacks(self, queue, handler, servers): + """ + @param queue Target worker queue + @see MumoManagerRemote + """ + return self.__rem_from_dict(self.metaCallbacks, queue, handler, servers) + + @local_thread + def subscribeServerCallbacks(self, queue, handler, servers): + """ + @param queue Target worker queue + @see MumoManagerRemote + """ + return self.__add_to_dict(self.serverCallbacks, queue, handler, servers) + + @local_thread + def unsubscribeServerCallbacks(self, queue, handler, servers): + """ + @param queue Target worker queue + @see MumoManagerRemote + """ + return self.__rem_from_dict(self.serverCallbacks, queue, handler, servers) + + @local_thread + def subscribeContextCallbacks(self, queue, handler, servers): + """ + @param queue Target worker queue + @see MumoManagerRemote + """ + return self.__add_to_dict(self.contextCallbacks, queue, handler, servers) + + @local_thread + def unsubscribeContextCallbacks(self, queue, handler, servers): + """ + @param queue Target worker queue + @see MumoManagerRemote + """ + return self.__rem_from_dict(self.contextCallbacks, queue, handler, servers) + + # + #--- Module load/start/stop/unload functionality + # + @local_thread_blocking + @debug_log(debug_me) + def loadModules(self, names = None): + """ + Loads a list of modules from the mumo directory structure by name. + + @param names List of names of modules to load + @return: List of modules loaded + """ + loadedmodules = {} + + if not names: + # If no names are given load all modules that have a configuration in the cfg_dir + if not os.path.isdir(self.cfg.modules.cfg_dir): + msg = "Module directory '%s' not found" % self.cfg.mumo.mod_dir + self.log().error(msg) + raise FailedLoadModuleImportException(msg) + + names = [] + for f in os.listdir(self.cfg.modules.cfg_dir): + if os.path.isfile(f): + base, ext = os.path.splitext(f) + if not ext or ext.tolower() == ".ini" or ext.tolower == ".conf": + names.append(base) + + for name in names: + try: + modinst = self._loadModule_noblock(name) + loadedmodules[name] = modinst + except FailedLoadModuleException: + pass + + return loadedmodules + + @local_thread_blocking + def loadModuleCls(self, name, modcls, module_cfg = None): + return self._loadModuleCls_noblock(name, modcls, module_cfg) + + @debug_log(debug_me) + def _loadModuleCls_noblock(self, name, modcls, module_cfg = None): + log = self.log() + + if name in self.modules: + log.error("Module '%s' already loaded", name) + return + + modqueue = Queue.Queue() + modmanager = MumoManagerRemote(self, name, modqueue) + + modinst = modcls(name, modmanager, module_cfg) + + # Remember it + self.modules[name] = modinst + self.queues[modqueue] = modinst + + return modinst + + @local_thread_blocking + def loadModule(self, name): + """ + Loads a single module either by name + + @param name Name of the module to load + @return Module instance + """ + self._loadModule_noblock(name) + + @debug_log(debug_me) + def _loadModule_noblock(self, name): + # Make sure this module is not already loaded + log = self.log() + log.debug("loadModuleByName('%s')", name) + + if name in self.modules: + log.warning("Tried to load already loaded module %s", name) + return + + # Check whether there is a configuration file for this module + confpath = self.cfg.modules.cfg_dir + name + '.ini' + if not os.path.isfile(confpath): + msg = "Module configuration file '%s' not found" % confpath + log.error(msg) + raise FailedLoadModuleConfigException(msg) + + # Make sure the module directory is in our python path and exists + if not self.cfg.mumo.mod_dir in sys.path: + if not os.path.isdir(self.cfg.mumo.mod_dir): + msg = "Module directory '%s' not found" % self.cfg.mumo.mod_dir + log.error(msg) + raise FailedLoadModuleImportException(msg) + sys.path.append(self.cfg.mumo.mod_dir) + + # Import the module and instanciate it + try: + mod = __import__(name) + self.imports[name] = mod + except ImportError, e: + msg = "Failed to import module '%s', reason: %s" % (name, str(e)) + log.error(msg) + raise FailedLoadModuleImportException(msg) + + try: + try: + modcls = mod.mumo_module_class # First check if there's a magic mumo_module_class variable + log.debug("Magic mumo_module_class found") + except AttributeError: + modcls = getattr(mod, name) + except AttributeError: + raise FailedLoadModuleInitializationException("Module does not contain required class %s" % name) + + return self._loadModuleCls_noblock(name, modcls, confpath) + + @local_thread_blocking + @debug_log(debug_me) + def startModules(self, names = None): + """ + Start a module by name + + @param names List of names of modules to start + @return A dict of started module names and instances + """ + log = self.log() + startedmodules = {} + + if not names: + # If no names are given start all models + names = self.modules.iterkeys() + + for name in names: + try: + modinst = self.modules[name] + if not modinst.is_alive(): + modinst.start() + log.debug("Module '%s' started", name) + else: + log.debug("Module '%s' already running", name) + startedmodules[name] = modinst + except KeyError: + log.error("Could not start unknown module '%s'", name) + + return startedmodules + + @local_thread_blocking + @debug_log(debug_me) + def stopModules(self, names = None, force = False): + """ + Stop a list of modules by name. Note that this only works + for well behaved modules. At this point if a module is really going + rampant you will have to restart mumo. + + @param names List of names of modules to unload + @param force Unload the module asap dropping messages queued for it + @return A dict of stopped module names and instances + """ + log = self.log() + stoppedmodules = {} + + if not names: + # If no names are given start all models + names = self.modules.iterkeys() + + for name in names: + try: + modinst = self.modules[name] + stoppedmodules[name] = modinst + except KeyError: + log.warning("Asked to stop unknown module '%s'", name) + continue + + if force: + # We will have to drain the modules queues + for queue, module in self.queues.iteritems(): + if module in self.modules: + try: + while queue.get_nowait(): pass + except Queue.Empty: pass + + for modinst in stoppedmodules.itervalues(): + if modinst.is_alive(): + modinst.stop() + log.debug("Module '%s' is being stopped", name) + else: + log.debug("Module '%s' already stopped", name) + + for modinst in stoppedmodules.itervalues(): + modinst.join(timeout = self.cfg.modules.timeout) + + return stoppedmodules + + def stop(self, force = True): + self.log().debug("Stopping") + self.stopModules() + Worker.stop(self, force) diff --git a/mumo_manager_test.py b/mumo_manager_test.py new file mode 100644 index 0000000..939142a --- /dev/null +++ b/mumo_manager_test.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import Queue +from mumo_manager import MumoManager, MumoManagerRemote +from mumo_module import MumoModule + + +class MumoManagerTest(unittest.TestCase): + def setUp(self): + class MyModule(MumoModule): + def __init__(self, name, manager, configuration = None): + MumoModule.__init__(self, name, manager, configuration) + self.was_called = False + self.par1 = None + self.par2 = None + self.par3 = None + + def last_call(self): + ret = (self.was_called, self.par1, self.par2, self.par3) + self.was_called = False + return ret + + def call_me(self, par1, par2 = None, par3 = None): + self.was_called = True + self.par1 = par1 + self.par2 = par2 + self.par3 = par3 + + self.man = MumoManager() + self.man.start() + + class conf(object): + pass # Dummy class + + cfg = conf() + cfg.test = 10 + + self.mod = self.man.loadModuleCls("MyModule", MyModule, cfg) + self.man.startModules() + + + + def tearDown(self): + self.man.stopModules() + self.man.stop() + self.man.join(timeout=2) + + + def testName(self): + pass + + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() \ No newline at end of file diff --git a/mumo_module.py b/mumo_module.py new file mode 100644 index 0000000..b753c24 --- /dev/null +++ b/mumo_module.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from config import Config, x2bool +from worker import Worker + +class MumoModule(Worker): + default_config = {} + + def __init__(self, name, manager, configuration = None): + Worker.__init__(self, name, manager.getQueue()) + self.__manager = manager + + if isinstance(configuration, basestring): + # If we are passed a string expect a config file there + if configuration: + self.__cfg = Config(configuration, self.default_config) + else: + self.__cfg = None + else: + # If we aren't passed a string it will be a config object or None + self.__cfg = configuration + + self.log().info("Initialized") + + #--- Accessors + def manager(self): + return self.__manager + + def cfg(self): + return self.__cfg + + #--- Module control + + + def onStart(self): + self.log().info("Start") + + def onStop(self): + self.log().info("Stop") + + #--- Events + + def connected(self): + # Called once the Ice connection to the murmur server + # is established. + # + # All event registration should happen here + + pass + + def disconnected(self): + # Called once a loss of Ice connectivity is detected. + # + + pass + + +def logModFu(fu): + def newfu(self, *args, **kwargs): + log = self.log() + argss = '' if len(args)==0 else ',' + ','.join(['"%s"' % str(arg) for arg in args]) + kwargss = '' if len(kwargs)==0 else ','.join('%s="%s"' % (kw, str(arg)) for kw, arg in kwargs.iteritems()) + log.debug("%s(%s%s%s)", fu.__name__, str(self), argss, kwargss) + return fu(self, *args, **kwargs) + return newfu \ No newline at end of file diff --git a/sketches.txt b/sketches.txt new file mode 100644 index 0000000..c912352 --- /dev/null +++ b/sketches.txt @@ -0,0 +1,83 @@ +******************** +* Folder structure * +******************** +@@@@@@@@@@ +/usr/sbin/ +@@@@@@@@@@ +mumo + +@@@@@@@@@ +/usr/lib/ +@@@@@@@@@ + +mumo/ + mumo.py + murmur_derivates.py + modules/ + idlemove.py + modbf2/ + __init__.py + engine.py + foo.py + bar.py + + +@@@@@ +/etc/ +@@@@@ + +mumo/ + mumo + modules-available/ + idlemove + modbf2 + modules-enabled/ + idlemove + + +********************* +* Config structure: * +********************* + +mumo +==== +[modules] +mod_dir = /usr/sbin/modules/ +cfg_dir = /etc/modules-enabled/ + +[Ice] +ip = 127.0.0.1 +port = 6502 +slice = Murmur.ice +secret = + +[logging] +file = /var/log/mumo/mumo.log +level = DEBUG + + +idlemove +======== +[idlemove] +servers = 1,2,3,4,6,7 +interval = 0.1 +timeout = 3600 +mute = True +deafen = False +channel = 123 + +modbf2 +====== +[modbf2] +bla = blub +blib = bernd + +[CoolServer1] +vserver = 1 +root = bf2gaming/CoolServer1Chan +sink = True + +[NotSoCoolServer] +vserver = 2 +root = NotSoCoolServerChan +sink = False diff --git a/test.py b/test.py new file mode 100644 index 0000000..c7135f5 --- /dev/null +++ b/test.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from worker_test import * +from config_test import * +from mumo_manager_test import * \ No newline at end of file diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..d5fce95 --- /dev/null +++ b/worker.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from threading import Thread +from Queue import Queue, Empty +from logging import getLogger + +def local_thread(fu): + """ + Decorator which makes a function execute in the local worker thread + Return values are discarded + """ + def new_fu(*args, **kwargs): + self = args[0] + self.message_queue().put((None, fu, args, kwargs)) + return new_fu + +def local_thread_blocking(fu, timeout = None): + """ + Decorator which makes a function execute in the local worker thread + The function will block until return values are available or timeout + seconds passed. + + @param timeout Timeout in seconds + """ + def new_fu(*args, **kwargs): + self = args[0] + out = Queue() + self.message_queue().put((out, fu, args, kwargs)) + ret, ex = out.get(True, timeout) + if ex: + raise ex + + return ret + + return new_fu + + +class Worker(Thread): + def __init__(self, name, message_queue = None): + """ + Implementation of a basic Queue based Worker thread. + + @param name Name of the thread to run the worker in + @param message_queue Message queue on which to receive commands + """ + + Thread.__init__(self, name = name) + self.daemon = True + self.__in = message_queue if message_queue != None else Queue() + self.__log = getLogger(name) + self.__name = name + + #--- Accessors + def log(self): + return self.__log + + def name(self): + return self.__name + + def message_queue(self): + return self.__in + + #--- Overridable convience stuff + def onStart(self): + """ + Override this function to perform actions on worker startup + """ + pass + + def onStop(self): + """ + Override this function to perform actions on worker shutdown + """ + pass + #--- Thread / Control + def run(self): + self.log().debug("Enter message loop") + self.onStart() + while True: + msg = self.__in.get() + if msg == None: + break + + (out, fu, args, kwargs) = msg + try: + res = fu(*args, **kwargs) + ex = None + except Exception, e: + self.log().exception(e) + res = None + ex = e + finally: + if not out is None: + out.put((res, ex)) + + self.onStop() + self.log().debug("Leave message loop") + + def stop(self, force = True): + if force: + try: + while True: + self.__in.get_nowait() + except Empty: + pass + + self.__in.put(None) + + #--- Helpers + + @local_thread + def call_by_name(self, handler, function_name, *args, **kwargs): + return getattr(handler, function_name)(*args, **kwargs) + + @local_thread_blocking + def call_by_name_blocking(self, handler, function_name, *args, **kwargs): + return getattr(handler, function_name)(*args, **kwargs) \ No newline at end of file diff --git a/worker_test.py b/worker_test.py new file mode 100644 index 0000000..a570f3a --- /dev/null +++ b/worker_test.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# -*- coding: utf-8 + +# Copyright (C) 2010 Stefan Hacker +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: + +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the Mumble Developers nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# `AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import worker +from worker import Worker, local_thread, local_thread_blocking +from Queue import Queue +from logging.handlers import BufferingHandler +from logging import ERROR +from threading import Event +from time import sleep + +class WorkerTest(unittest.TestCase): + def setUp(self): + + def set_ev(fu): + def new_fu(*args, **kwargs): + s = args[0] + s.event.set() + s.val = (args, kwargs) + return fu(*args, **kwargs) + return new_fu + + class ATestWorker(Worker): + def __init__(self, name, message_queue): + Worker.__init__(self, name, message_queue) + self.event = Event() + self.val = None + self.started = False + self.stopped = False + + @local_thread + @set_ev + def echo(self, val): + return val + + @local_thread_blocking + @set_ev + def echo_block(self, val): + return val + + def onStart(self): + self.started = True + + def onStop(self): + self.stopped = True + + @local_thread + def raise_(self, ex): + raise ex + + @local_thread_blocking + def raise_blocking(self, ex): + raise ex + + @set_ev + def call_me_by_name(self, arg1, arg2): + return + + def call_me_by_name_blocking(self, arg1, arg2): + return arg1, arg2 + + + self.buha = BufferingHandler(10000) + + q = Queue() + self.q = q + self.w = ATestWorker("Test", q) + l = self.w.log() + l.addHandler(self.buha) + assert(self.w.started == False) + self.w.start() + sleep(0.05) + assert(self.w.started == True) + + def testName(self): + assert(self.w.name() == "Test") + + def testMessageQueue(self): + assert(self.w.message_queue() == self.q) + + def testLocalThread(self): + s = "Testing" + self.w.event.clear() + self.w.echo(s) + self.w.event.wait(5) + args, kwargs = self.w.val + + assert(args[1] == s) + + def testLocalThreadException(self): + self.buha.flush() + self.w.raise_(Exception()) + sleep(0.1) # hard delay + assert(len(self.buha.buffer) != 0) + assert(self.buha.buffer[0].levelno == ERROR) + + def testCallByName(self): + self.w.event.clear() + self.w.call_by_name(self.w, "call_me_by_name", "arg1", arg2="arg2") + self.w.event.wait(5) + args, kwargs = self.w.val + + assert(args[1] == "arg1") + assert(kwargs["arg2"] == "arg2") + + def testLocalThreadBlocking(self): + s = "Testing" + assert(s == self.w.echo_block(s)) + + def testLocalThreadExceptionBlocking(self): + class TestException(Exception): pass + self.assertRaises(TestException, self.w.raise_blocking, TestException()) + + def testCallByNameBlocking(self): + arg1, arg2 = self.w.call_by_name_blocking(self.w, "call_me_by_name_blocking", "arg1", arg2="arg2") + + assert(arg1 == "arg1") + assert(arg2 == "arg2") + + def tearDown(self): + assert(self.w.stopped == False) + self.w.stop() + self.w.join(5) + assert(self.w.stopped == True) + + + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + unittest.main() \ No newline at end of file