Source code for cis_interface.interface.PsiInterface

from logging import debug
import os
import time
from sysv_ipc import MessageQueue
from cis_interface.backwards import pickle
from cis_interface.interface.scanf import scanf
from cis_interface.dataio.AsciiFile import AsciiFile
from cis_interface.dataio.AsciiTable import AsciiTable
from cis_interface import backwards


# OS X limit is 2kb
PSI_MSG_MAX = 1024 * 2
PSI_MSG_EOF = backwards.unicode2bytes("EOF!!!")


[docs]def PsiMatlab(_type, args): r"""Short interface to identify functions called by Matlab. Args: _type (str): Name of class that should be returned. args (list): Additional arguments that should be passed to class initializer. Returns: obj: An instance of the requested class. """ cls = eval(_type) kwargs = {'matlab': True} obj = cls(*args, **kwargs) return obj
[docs]class PsiInput(object): r"""Class for handling input from a message queue. Args: name (str): The name of the message queue. Combined with the suffix '_INT', it should match an environment variable containing a message queue key. Attributes: name (str): The name of the message queue. qname (str): The name of the message queue combined with the suffix '_IN'. q (:class:`sysv_ipc.MessageQueue`): Message queue. """ name = None qName = None q = None def __init__(self, name, matlab=False): self.name = name self.qName = name + '_IN' self.q = None debug("PsiInput(%s):", name) if self.qName not in os.environ: raise Exception('PsiInterface cant see %s in env.' % self.qName) # print('ERROR: PsiInterface cant see ' + name + ' in env') # exit(-1) qid = os.environ.get(self.qName, '') qid = int(qid) debug("PsiInput(%s): qid %s", self.name, qid) self.q = MessageQueue(qid, max_message_size=PSI_MSG_MAX) clidebug = os.environ.get('PSI_CLIENT_DEBUG', False) self.sleeptime = 0.25 if clidebug: self.sleeptime = 2.0
[docs] def recv(self): r"""Receive a message smaller than PSI_MSG_MAX. The process will sleep until there is a message in the queue to receive. Returns: tuple (bool, str): The success or failure of receiving a message and the message received. """ payload = (False, '') debug("PsiInput(%s).recv()", self.name) try: while self.q.current_messages == 0: debug("PsiInput(%s): recv() - no data, sleep", self.name) time.sleep(self.sleeptime) debug("PsiInput(%s).recv(): message ready, read it", self.name) data, _ = self.q.receive() # ignore ident payload = (True, data) debug("PsiInput(%s).recv(): read %d bytes", self.name, len(data)) except Exception as ex: # pragma: debug debug("PsiInput(%s).recv(): exception %s, return None", self.name, type(ex)) return payload
[docs] def recv_nolimit(self): r"""Receive a message larger than PSI_MSG_MAX that is sent in multiple parts. Returns: tuple (bool, str): The success or failure of receiving a message and the complete message received. """ debug("PsiInput(%s).recv_nolimit()", self.name) payload = self.recv() if not payload[0]: # pragma: debug debug("PsiInput(%s).recv_nolimit(): " + "Failed to receive payload size.", self.name) return payload try: # (leng_exp,) = scanf('%d', payload[1]) leng_exp = int(float(payload[1])) data = '' ret = True while len(data) < leng_exp: payload = self.recv() if not payload[0]: # pragma: debug debug("PsiInput(%s).recv_nolimit(): " + "read interupted at %d of %d bytes.", self.name, len(data), leng_exp) ret = False break data += payload[1] payload = (ret, data) debug("PsiInput(%s).recv_nolimit(): read %d bytes", self.name, len(data)) except Exception as ex: # pragma: debug payload = (False, True) debug("PsiInput(%s).recv_nolimit(): " + "exception %s, return None", self.name, type(ex)) return payload
[docs]class PsiOutput: r"""Class for handling output to a message queue. Args: name (str): The name of the message queue. Combined with the suffix '_OUT', it should match an environment variable containing a message queue key. Attributes: name (str): The name of the message queue. qname (str): The name of the message queue combined with the suffix '_OUT'. q (:class:`sysv_ipc.MessageQueue`): Message queue. """ def __init__(self, name, matlab=False): self.name = name self.qName = name + '_OUT' debug("PsiOputput(%s)", name) if self.qName not in os.environ: raise Exception('PsiInterface cant see %s in env.' % self.qName) # print('ERROR: PsiInterface cant see ' + name + ' in env') # exit(-1) qid = int(os.environ[name + '_OUT']) self.q = MessageQueue(qid, max_message_size=PSI_MSG_MAX) return
[docs] def send(self, payload): r"""Send a message smaller than PSI_MSG_MAX. Args: payload (str): Message to send. Returns: bool: Success or failure of sending the message. """ ret = False try: debug("PsiOutput(%s).send(%s)", self.name, payload) self.q.send(payload) ret = True debug("PsiOutput(%s).sent(%s)", self.name, payload) except Exception as ex: # pragma: debug debug("PsiOutput(%s).send(%s): exception: %s", self.name, payload, type(ex)) debug("PsiOutput(%s).send(%s): returns %d", self.name, payload, ret) return ret
[docs] def send_nolimit(self, payload): r"""Send a message larger than PSI_MSG_MAX in multiple parts. Args: payload (str): Message to send. Returns: bool: Success or failure of sending the message. """ ret = self.send("%ld" % len(payload)) if not ret: # pragma: debug debug("PsiOutput(%s).send_nolimit: " + "Sending size of payload failed.", self.name) return ret prev = 0 while prev < len(payload): next = min(prev + PSI_MSG_MAX, len(payload)) ret = self.send(payload[prev:next]) if not ret: # pragma: debug debug("PsiOutput(%s).send_nolimit(): " + "send interupted at %d of %d bytes.", self.name, prev, len(payload)) break debug("PsiOutput(%s).send_nolimit(): %d of %d bytes sent", self.name, next, len(payload)) prev = next if ret: debug("PsiOutput(%s).send_nolimit %d bytes completed", self.name, len(payload)) return ret
[docs]class PsiRpc(object): r"""Class for sending a message and then receiving a response. Args: outname (str): The name of the output message queue. outfmt (str): Format string used to format variables in a message sent to the output message queue. inname (str): The name of the input message queue. infmt (str): Format string used to recover variables from messages received from the input message queue. """ def __init__(self, outname, outfmt, inname, infmt, matlab=False): if matlab: self._inFmt = backwards.decode_escape(infmt) self._outFmt = backwards.decode_escape(outfmt) else: self._inFmt = infmt self._outFmt = outfmt self._in = PsiInput(inname) self._out = PsiOutput(outname)
[docs] def rpcSend(self, *args): r"""Send arguments as a message created using the output format string. Args: \*args: All arguments are formatted using the output format string to create the message. Returns: bool: Success or failure of sending the message. """ outmsg = self._outFmt % args return self._out.send_nolimit(outmsg)
[docs] def rpcRecv(self): r"""Receive a message and get arguments by parsing the recieved message using the input format string. Returns: tuple (bool, tuple): Success or failure of receiving a message and the tuple of arguments retreived by parsing the message using the input format string. """ retval, args = self._in.recv_nolimit() if retval: args = scanf(self._inFmt, args) return retval, args
[docs] def rpcCall(self, *args): r"""Send arguments using the output format string to format a message and then receive values back by parsing the response message with the input format string. Args: \*args: All arguments are formatted using the output format string to create the message. Returns: tuple (bool, tuple): Success or failure of receiving a message and the tuple of arguments retreived by parsing the message using the input format string. """ ret = self.rpcSend(*args) if ret: return self.rpcRecv()
[docs]class PsiRpcServer(PsiRpc): r"""Class for handling requests and response for an RPC Server. Args: name (str): The name of the server queues. infmt (str): Format string used to recover variables from messages received from the request queue. outfmt (str): Format string used to format variables in a message sent to the response queue. """ def __init__(self, name, infmt, outfmt, matlab=False): super(PsiRpcServer, self).__init__(name, outfmt, name, infmt)
[docs]class PsiRpcClient(PsiRpc): r"""Class for handling requests and response to an RPC Server from a client. Args: name (str): The name of the server queues. outfmt (str): Format string used to format variables in a message sent to the request queue. infmt (str): Format string used to recover variables from messages received from the response queue. """ def __init__(self, name, outfmt, infmt, matlab=False): super(PsiRpcClient, self).__init__(name, outfmt, name, infmt)
# Specialized classes for ascii IO
[docs]class PsiAsciiFileInput(object): r"""Class for generic ASCII input from either a file or message queue. Args: name (str): Path to the local file that input should be read from (if src_type == 0) or the name of the input message queue that input should be received from. src_type (int, optional): If 0, input is read from a local file. Otherwise input is received from a message queue. Defauts to 1. """ _name = None _type = 0 _file = None _psi = None def __init__(self, name, src_type=1, matlab=False): self._name = name self._type = src_type if self._type == 0: self._file = AsciiFile(name, 'r') self._file.open() else: self._psi = PsiInput(name) self._file = AsciiFile(name, None) def __del__(self): if self._type == 0 and (self._file is not None): self._file.close() self._file = None
[docs] def recv_line(self): r"""Receive a single line of ASCII input. Returns: tuple(bool, str): Success or failure of receiving a line and the line received (including the newline character). """ if self._type == 0: eof, line = self._file.readline() ret = (not eof) else: ret, line = self._psi.recv() if (len(line) == 0) or (line == PSI_MSG_EOF): ret = False return ret, line
[docs]class PsiAsciiFileOutput(object): r"""Class for generic ASCII output to either a local file or message queue. Args: name (str): Path to the local file where output should be written (if dst_type == 0) or the name of the message queue where output should be sent. dst_type (int, optional): If 0, output is written to a local file. Otherwise, output is sent to a message queue. Defaults to 1. """ _name = None _type = 0 _file = None _psi = None def __init__(self, name, dst_type=1, matlab=False): self._name = name self._type = dst_type if self._type == 0: self._file = AsciiFile(name, 'w') self._file.open() else: self._psi = PsiOutput(name) self._file = AsciiFile(name, None) def __del__(self): if self._type == 0 and (self._file is not None): self._file.close() self._file = None
[docs] def send_eof(self): r"""Send an end-of-file message to the message queue.""" if self._type == 0: pass else: self._psi.send(PSI_MSG_EOF)
[docs] def send_line(self, line): r"""Output a single ASCII line. Args: line (str): Line to output (including newline character). Returns: bool: Success or failure of sending the line. """ if self._type == 0: self._file.writeline(line) ret = True else: ret = self._psi.send(line) return ret
# Specialized classes for ascii table IO
[docs]class PsiAsciiTableInput(object): r"""Class for handling table-like formatted input. Args: name (str): The path to the local file to read input from (if src_type == 0) or the name of the message queue input should be received from. src_type (int, optional): If 0, input is read from a local file. Otherwise, the input is received from a message queue. Defaults to 1. """ _name = None _type = 0 _table = None _psi = None def __init__(self, name, src_type=1, matlab=False): self._name = name self._type = src_type if self._type == 0: self._table = AsciiTable(name, 'r') self._table.open() else: self._psi = PsiInput(name) ret, format_str = self._psi.recv() if not ret: # pragma: debug print('ERROR: PsiAsciiTableInput could not ' + 'receive format string from input') exit(-1) self._table = AsciiTable( name, None, format_str=format_str.decode('string_escape')) def __del__(self): if self._type == 0: self._table.close()
[docs] def recv_row(self): r"""Receive a single row of variables from the input. Returns: tuple(bool, tuple): Success or failure of receiving the row and the variables recovered from the row. """ if self._type == 0: eof, args = self._table.readline() ret = (not eof) else: ret, args = self._psi.recv_nolimit() if ret: args = self._table.process_line(args) if args is None: ret = False return ret, args
[docs] def recv_array(self): r"""Receive an entire array of table data. Returns: tuple(bool, np.ndarray): Success or failure of receiving the row and the array of table data. """ if self._type == 0: arr = self._table.read_array() ret = True else: ret, data = self._psi.recv_nolimit() if ret: arr = self._table.bytes_to_array(data, order='F') if arr is None: # pragma: debug ret = False else: # pragma: debug arr = None return ret, arr
[docs]class PsiAsciiTableOutput(object): r"""Class for handling table-like formatted output. Args: name (str): The path to the local file where output should be saved (if dst_type == 0) or the name of the message queue where the output should be sent. fmt (str): A C style format string specifying how each 'row' of output should be formated. This should include the newline character. dst_type (int, optional): If 0, output is sent to a local file. Otherwise, the output is sent to a message queue. Defaults to 1. """ _name = None _type = 0 _table = None _psi = None def __init__(self, name, fmt, dst_type=1, matlab=False): self._name = name self._type = dst_type if matlab: fmt = backwards.decode_escape(fmt) if self._type == 0: self._table = AsciiTable(name, 'w', format_str=fmt) self._table.open() self._table.writeformat() else: self._psi = PsiOutput(name) self._table = AsciiTable(name, None, format_str=fmt) self._psi.send(fmt.decode('string_escape')) def __del__(self): if self._type == 0: self._table.close()
[docs] def send_eof(self): r"""Send an end-of-file message to the message queue.""" if self._type == 0: pass else: self._psi.send_nolimit(PSI_MSG_EOF)
[docs] def send_row(self, *args): r"""Output arguments as a formated row to either a local file or message queue. Args: \*args: All arguments are formated to create a table 'row'. Returns: bool: Success or failure of outputing the row. """ if (len(args) == 1) and isinstance(args[0], tuple): args = args[0] if self._type == 0: self._table.writeline(*args) ret = True else: msg = self._table.format_line(*args) ret = self._psi.send_nolimit(msg) return ret
[docs] def send_array(self, arr): r"""Output an array of table data to either a local file or message sueue. Args: arr (numpy.ndarray): Array of table data. The first dimension is assumed to be table rows and the second dimension is assumed to be table columns. Returns: bool: Success or failure of outputing the array. """ if self._type == 0: self._table.write_array(arr, skip_header=False) ret = True else: msg = self._table.array_to_bytes(arr, order='F') ret = self._psi.send_nolimit(msg) return ret
[docs]class PsiPickleInput(object): r"""Class for handling pickled input. Args: name (str): The path to the local file to read input from (if src_type == 0) or the name of the message queue input should be received from. src_type (int, optional): If 0, input is read from a local file. Otherwise, the input is received from a message queue. Defaults to 1. """ _name = None _type = 1 _file = None _psi = None def __init__(self, name, src_type=1, matlab=False): self._name = name self._type = src_type if self._type == 0: self._file = open(name, 'rb') else: self._psi = PsiInput(name) def __del__(self): if self._type == 0 and (self._file is not None): self._file.close() self._file = None
[docs] def recv(self): r"""Receive a single pickled object. Returns: tuple(bool, object): Success or failure of receiving a pickled object and the unpickled object that was received. """ if self._type == 0: try: obj = pickle.load(self._file) eof = False except EOFError: # pragma: debug obj = None eof = True ret = (not eof) else: ret, obj = self._psi.recv_nolimit() try: obj = pickle.loads(obj) except pickle.UnpicklingError: # pragma: debug obj = None ret = False return ret, obj
[docs]class PsiPickleOutput(object): r"""Class for handling pickled output. Args: name (str): The path to the local file where output should be saved (if dst_type == 0) or the name of the message queue where the output should be sent. fmt (str): A C style format string specifying how each 'row' of output should be formated. This should include the newline character. dst_type (int, optional): If 0, output is sent to a local file. Otherwise, the output is sent to a message queue. Defaults to 1. """ _name = None _type = 0 _file = None _psi = None def __init__(self, name, dst_type=1, matlab=False): self._name = name self._type = dst_type if self._type == 0: self._file = open(name, 'wb') else: self._psi = PsiOutput(name) def __del__(self): if self._type == 0 and (self._file is not None): self._file.close() self._file = None
[docs] def send(self, obj): r"""Output an object as a pickled string to either a local file or message queue. Args: obj (object): Any python object that can be pickled. Returns: bool: Success or failure of outputing the pickled object. """ if self._type == 0: try: pickle.dump(obj, self._file) ret = True except pickle.PicklingError: # pragma: debug ret = False else: try: msg = pickle.dumps(obj) ret = True except pickle.PicklingError: # pragma: debug ret = False if ret: ret = self._psi.send_nolimit(msg) return ret