Source code for deephaven.table_listener

#
# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
#

""" This module provides utilities for listening to table changes. """
from __future__ import annotations

from abc import ABC, abstractmethod
from functools import wraps
from inspect import signature
from typing import Callable, Union, List, Generator, Dict, Optional, Literal

import jpy
import numpy

from deephaven import DHError
from deephaven import update_graph
from deephaven._wrapper import JObjectWrapper
from deephaven.column import Column
from deephaven.jcompat import to_sequence
from deephaven.numpy import column_to_numpy_array
from deephaven.table import Table
from deephaven.update_graph import UpdateGraph

_JPythonReplayListenerAdapter = jpy.get_type("io.deephaven.integrations.python.PythonReplayListenerAdapter")
_JTableUpdate = jpy.get_type("io.deephaven.engine.table.TableUpdate")
_JTableUpdateDataReader = jpy.get_type("io.deephaven.integrations.python.PythonListenerTableUpdateDataReader")


def _col_defs(table: Table, cols: Union[str, List[str]]) -> List[Column]:
    if not cols:
        col_defs = table.columns
    else:
        cols = to_sequence(cols)
        col_defs = [col for col in table.columns if col.name in cols]

    return col_defs


def _changes_to_numpy(table: Table, cols: Union[str, List[str]], row_set, chunk_size: Optional[int],
                      prev: bool = False) -> Generator[Dict[str, numpy.ndarray], None, None]:
    col_defs = _col_defs(table, cols)

    row_sequence_iterator = row_set.getRowSequenceIterator()
    col_sources = [table.j_table.getColumnSource(col_def.name) for col_def in col_defs]
    chunk_size = row_set.size() if not chunk_size else chunk_size
    j_reader_context = _JTableUpdateDataReader.makeContext(chunk_size, *col_sources)
    try:
        while row_sequence_iterator.hasMore():
            chunk_row_set = row_sequence_iterator.getNextRowSequenceWithLength(chunk_size)

            j_array = _JTableUpdateDataReader.readChunkColumnMajor(j_reader_context, chunk_row_set, col_sources, prev)

            col_dict = {}
            for i, col_def in enumerate(col_defs):
                np_array = column_to_numpy_array(col_def, j_array[i])
                col_dict[col_def.name] = np_array

            yield col_dict
    finally:
        j_reader_context.close()
        row_sequence_iterator.close()


[docs]class TableUpdate(JObjectWrapper): j_object_type = _JTableUpdate def __init__(self, table: Table, j_table_update: jpy.JType): self.table = table self.j_table_update = j_table_update @property def j_object(self) -> jpy.JType: return self.j_table_update
[docs] def added(self, cols: Union[str, List[str]] = None) -> Dict[str, numpy.ndarray]: """Returns a dict with each key being a column name and each value being a NumPy array of all the added rows in the columns. Args: cols (Union[str, List[str]): the column(s) for which to return the added rows Returns: a dict """ if not self.j_table_update.added: return {} try: return next( _changes_to_numpy(table=self.table, cols=cols, row_set=self.j_table_update.added.asRowSet(), chunk_size=None)) except StopIteration: return {}
[docs] def added_chunks(self, chunk_size: int, cols: Union[str, List[str]] = None) -> Generator[ Dict[str, numpy.ndarray], None, None]: """Returns a generator that on each iteration, only returns a chunk of added rows in the form of a dict with each key being a column name and each value being a NumPy array of the rows in the chunk. Args: chunk_size (int): the size of the chunk cols (Union[str, List[str]]): the columns(s) for which to return the added rows Returns: a generator """ if not self.j_table_update.added: return (_ for _ in ()) return _changes_to_numpy(table=self.table, cols=cols, row_set=self.j_table_update.added.asRowSet(), chunk_size=chunk_size)
[docs] def removed(self, cols: Union[str, List[str]] = None) -> Dict[str, numpy.ndarray]: """Returns a dict with each key being a column name and each value being a NumPy array of all the removed rows in the columns. Args: cols (Union[str, List[str]): the column(s) for which to return the added rows Returns: a dict """ if not self.j_table_update.removed: return {} try: return next( _changes_to_numpy(table=self.table, cols=cols, row_set=self.j_table_update.removed.asRowSet(), chunk_size=None, prev=True)) except StopIteration: return {}
[docs] def removed_chunks(self, chunk_size: int, cols: Union[str, List[str]] = None) -> Generator[ Dict[str, numpy.ndarray], None, None]: """Returns a generator that on each iteration, only returns a chunk of removed rows in the form of a dict with each key being a column name and each value being a NumPy array of the rows in the chunk. Args: chunk_size (int): the size of the chunk cols (Union[str, List[str]]): the columns(s) for which to return the added rows Returns: a generator """ if not self.j_table_update.removed: return (_ for _ in ()) return _changes_to_numpy(table=self.table, cols=cols, row_set=self.j_table_update.removed.asRowSet(), chunk_size=chunk_size, prev=True)
[docs] def modified(self, cols: Union[str, List[str]] = None) -> Dict[str, numpy.ndarray]: """Returns a dict with each key being a column name and each value being a NumPy array of the current values of all the modified rows in the columns. Args: cols (Union[str, List[str]): the column(s) for which to return the added rows Returns: a dict """ if not self.j_table_update.modified: return {} try: return next( _changes_to_numpy(self.table, cols=cols, row_set=self.j_table_update.modified.asRowSet(), chunk_size=None)) except StopIteration: return {}
[docs] def modified_chunks(self, chunk_size: int, cols: Union[str, List[str]] = None) -> Generator[ Dict[str, numpy.ndarray], None, None]: """Returns a generator that on each iteration, only returns a chunk of modified rows in the form of a dict with each key being a column name and each value being a NumPy array of the current values of the rows in the chunk. Args: chunk_size (int): the size of the chunk cols (Union[str, List[str]]): the columns(s) for which to return the added rows Returns: a generator """ if not self.j_table_update.modified: return (_ for _ in ()) return _changes_to_numpy(self.table, cols=cols, row_set=self.j_table_update.modified.asRowSet(), chunk_size=chunk_size)
[docs] def modified_prev(self, cols: Union[str, List[str]] = None) -> Dict[str, numpy.ndarray]: """Returns a dict with each key being a column name and each value being a NumPy array of the previous values of all the modified rows in the columns. Args: cols (Union[str, List[str]): the column(s) for which to return the added rows Returns: a dict """ if not self.j_table_update.modified: return {} try: return next( _changes_to_numpy(self.table, cols=cols, row_set=self.j_table_update.modified.asRowSet(), chunk_size=None, prev=True)) except StopIteration: return {}
[docs] def modified_prev_chunks(self, chunk_size: int, cols: Union[str, List[str]] = None) -> Generator[ Dict[str, numpy.ndarray], None, None]: """Returns a generator that on each iteration, only returns a chunk of modified rows in the form of a dict with each key being a column name and each value being a NumPy array of the previous values of the rows in the chunk. Args: chunk_size (int): the size of the chunk cols (Union[str, List[str]]): the columns(s) for which to return the added rows Returns: a generator """ if not self.j_table_update.modified: return (_ for _ in ()) return _changes_to_numpy(self.table, cols=cols, row_set=self.j_table_update.modified.asRowSet(), chunk_size=chunk_size, prev=True)
@property def shifted(self): raise NotImplemented("shifts are not supported yet.") @property def modified_columns(self) -> List[str]: """The list of modified columns in this update.""" cols = self.j_table_update.modifiedColumnSet.dirtyColumnNames() return list(cols) if cols else []
def _do_locked(ug: Union[UpdateGraph, Table], f: Callable, lock_type: Literal["shared","exclusive"] = "shared") -> \ None: """Executes a function while holding the UpdateGraph (UG) lock. Holding the UG lock ensures that the contents of a table will not change during a computation, but holding the lock also prevents table updates from happening. The lock should be held for as little time as possible. Args: ug (Union[UpdateGraph, Table]): The Update Graph (UG) or a table-like object. f (Callable): callable to execute while holding the UG lock, could be function or an object with an 'apply' attribute which is callable lock_type (str): UG lock type, valid values are "exclusive" and "shared". "exclusive" allows only a single reader or writer to hold the lock. "shared" allows multiple readers or a single writer to hold the lock. Raises: ValueError """ if isinstance(ug, Table): ug = ug.update_graph if lock_type == "exclusive": with update_graph.exclusive_lock(ug): f() elif lock_type == "shared": with update_graph.shared_lock(ug): f() else: raise ValueError(f"Unsupported lock type: lock_type={lock_type}")
[docs]class TableListener(ABC): """An abstract table listener class that should be subclassed by any user table listener class."""
[docs] @abstractmethod def on_update(self, update: TableUpdate, is_replay: bool) -> None: """The required method on a listener object that receives table updates.""" ...
def _listener_wrapper(table: Table): """A decorator to wrap a user listener function or on_update method to receive the numpy-converted Table updates. Args: table (Table): the table to listen for updates. """ def decorator(listener: Callable): @wraps(listener) def wrapper(update, *args): t_update = TableUpdate(table=table, j_table_update=update) listener(t_update, args[0]) return wrapper return decorator def _wrap_listener_func(t: Table, listener: Callable): n_params = len(signature(listener).parameters) if n_params != 2: raise ValueError("listener function must have 2 (update, is_replay) parameters.") return _listener_wrapper(table=t)(listener) def _wrap_listener_obj(t: Table, listener: TableListener): n_params = len(signature(listener.on_update).parameters) if n_params != 2: raise ValueError(f"The on_update method must have 2 (update, is_replay) parameters.") listener.on_update = _listener_wrapper(table=t)(listener.on_update) return listener
[docs]def listen(t: Table, listener: Union[Callable, TableListener], description: str = None, do_replay: bool = False, replay_lock: Literal["shared", "exclusive"] = "shared"): """This is a convenience function that creates a TableListenerHandle object and immediately starts it to listen for table updates. The function returns the created TableListenerHandle object whose 'stop' method can be called to stop listening. If it goes out of scope and is garbage collected, the listener will stop receiving any table updates. Args: t (Table): table to listen to listener (Union[Callable, TableListener]): listener for table changes description (str, optional): description for the UpdatePerformanceTracker to append to the listener's entry description, default is None do_replay (bool): whether to replay the initial snapshot of the table, default is False replay_lock (str): the lock type used during replay, default is 'shared', can also be 'exclusive' Returns: a TableListenerHandle Raises: DHError """ table_listener_handle = TableListenerHandle(t=t, listener=listener, description=description) table_listener_handle.start(do_replay=do_replay, replay_lock=replay_lock) return table_listener_handle
[docs]class TableListenerHandle(JObjectWrapper): """A handle to manage a table listener's lifecycle.""" j_object_type = _JPythonReplayListenerAdapter def __init__(self, t: Table, listener: Union[Callable, TableListener], description: str = None): """Creates a new table listener handle. Table change events are processed by 'listener', which can be either (1) a callable (e.g. function) or (2) an instance of TableListener type which provides an "on_update" method. The callable or the on_update method must have the following signatures. * (update: TableUpdate, is_replay: bool): support replaying the initial table snapshot and normal table updates The 'update' parameter is an object that describes the table update; The 'is_replay' parameter is used only by replay listeners, it is set to 'true' when replaying the initial snapshot and 'false' during normal updates. Args: t (Table): table to listen to listener (Union[Callable, TableListener]): listener for table changes description (str, optional): description for the UpdatePerformanceTracker to append to the listener's entry description, default is None Raises: ValueError """ self.t = t if callable(listener): listener_wrapped = _wrap_listener_func(t, listener) elif isinstance(listener, TableListener): listener_wrapped = _wrap_listener_obj(t, listener) else: raise ValueError("listener is neither callable nor TableListener object") self.listener = _JPythonReplayListenerAdapter(description, t.j_table, False, listener_wrapped) self.started = False
[docs] def start(self, do_replay: bool = False, replay_lock: Literal["shared", "exclusive"] = "shared") -> None: """Start the listener by registering it with the table and listening for updates. Args: do_replay (bool): whether to replay the initial snapshot of the table, default is False replay_lock (str): the lock type used during replay, default is 'shared', can also be 'exclusive'. Raises: DHError """ if self.started: raise RuntimeError("Attempting to start an already started listener..") try: def _start(): if do_replay: self.listener.replay() self.t.j_table.addUpdateListener(self.listener) if do_replay: _do_locked(self.t, _start, lock_type=replay_lock) else: _start() except Exception as e: raise DHError(e, "failed to listen to the table changes.") from e self.started = True
[docs] def stop(self) -> None: """Stop the listener by de-registering it from the table and stop listening for updates.""" if not self.started: return self.t.j_table.removeUpdateListener(self.listener) self.started = False
@property def j_object(self) -> jpy.JType: return self.listener