How to use Deephaven in a local development environment

import com.fishlib.auth.SimpleUserContext;
import com.fishlib.configuration.Configuration;
import com.fishlib.io.logger.Logger;
import com.fishlib.io.logger.StreamLoggerImpl;
import com.illumon.iris.db.schema.SchemaService;
import com.illumon.iris.db.schema.SchemaServiceFactory;
import com.illumon.iris.db.tables.Table;
import com.illumon.iris.db.tables.databases.Database;
import com.illumon.iris.db.tables.databases.OnDiskQueryDatabase;
import com.illumon.iris.db.tables.utils.TableTools;
import com.illumon.iris.db.v2.locations.TableDataService;
import com.illumon.iris.db.v2.locations.local.LocalTableDataService;
import com.illumon.iris.db.v2.locations.local.LocalTablePathManager;
import com.illumon.iris.db.v2.permissions.PermissionFilterProvider;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;

public class UnitTest {

   private static final Logger logger = new StreamLoggerImpl();
   private static final String TEST_DATE = "2017-08-25";

   private static Database getTestDatabase() throws IOException {
      //create a local SchemaService
      final Configuration configuration = Configuration.getInstance();
      final String schemaDir = "src/test/resources";
      final SchemaService schemaService = SchemaServiceFactory.getLocalInstance(configuration, schemaDir);

      //create the OnDiskQueryDatabase
      final Path rootPath = Files.createTempDirectory(Paths.get(Configuration.getInstance().getWorkspacePath()), "UnitTest");
      final File rootFile = rootPath.toFile();
      final LocalTablePathManager pathManager = new LocalTablePathManager(rootFile);
      final TableDataService tableDataService = new LocalTableDataService(pathManager);
      final OnDiskQueryDatabase db = new OnDiskQueryDatabase(logger, rootFile, tableDataService, schemaService);

      //start the OnDiskQueryDatabase and set a user context
      db.startup();
      Configuration.getInstance().setProperty("IrisDB.permissionFilterProvider", "null");
      PermissionFilterProvider.FACTORY.reload();
      db.setUserContext(null, new SimpleUserContext("nobody", "nobody"));

      //add the test data
      addData(db);
      return db;
   }

   private static void addData(final Database db) throws IOException {
      final Table t = TableTools.readCsv(new File("src/test/resources/StockTrades.csv"));
      db.addTablePartition("Example", "StockTrades", TEST_DATE, t);
   }

   @Test
   public void testQuery() throws IOException {
      final Database db = getTestDatabase();

      final Table testResult = QueryUtils.getAvgStockPriceTable(db, TEST_DATE);

      final double sumAAPL = testResult.where("USym = `AAPL`").getColumn("Last").getDouble(0);
      Assertions.assertEquals(159.472, sumAAPL, 0.0002);

      final double sumGOOG = testResult.where("USym = `GOOG`").getColumn("Last").getDouble(0);
      Assertions.assertEquals(923.56, sumGOOG, 0.0002);
   }
}

import com.fishlib.configuration.Configuration;
import com.fishlib.io.log.LogLevel;
import com.fishlib.io.logger.DelegatingLoggerImpl;
import com.fishlib.io.logger.Logger;
import com.fishlib.logger.ProcessUnsafeLogCreator;
import com.fishlib.net.impl.nio.NIODriver;
import com.fishlib.util.process.ProcessEnvironment;
import com.illumon.iris.db.tables.Table;
import com.illumon.iris.db.tables.databases.Database;
import com.illumon.iris.db.tables.remote.RemoteDatabase;
import com.illumon.iris.db.tables.remotequery.ContextAwareRemoteQuery;
import com.illumon.iris.db.tables.remotequery.RemoteProcessingRequest;
import com.illumon.iris.db.tables.remotequery.RemoteQueryClient;
import com.illumon.iris.db.tables.remotequery.RemoteQueryUtils;
import com.illumon.iris.db.tables.utils.DBTimeUtils;
import com.illumon.iris.db.tables.utils.TableTools;
import org.jetbrains.annotations.NotNull;

import java.io.IOException;
import java.io.UncheckedIOException;

/**
 * Simple example for executing a {@link ContextAwareRemoteQuery} against a Deephaven intraday table and partition.
 *
 * This example creates a Deephaven database connection (RemoteDatabase), and also creates an inner
 * query class to allow the query steps to be executed as a single request to the query server.
 * The query class extends ContextAwareRemoteQuery, and an instance of this class is then passed
 * to the RemoteDatabase executeQuery method to execute the contained query and return the result.
 *
 * For a simpler example that uses ad-hoc method calls against the RemoteDatabase object to execute
 * the same query, see RemoteDatabaseExample.java.
 */
public class CARQExample {
    private static final String PROCESS_NAME = "CARQExample";

    private static void prepareEnvironment() {
        // the Configuration object reads the property file, which has named key-value pairs that control the
        // operation of the system.
        final Configuration configuration = Configuration.getInstance();

        // the log object is used by Deephaven components, and produces output in a PROCESS_NAME.log.<date> with
        // a hard link to PROCESS_NAME.log.current (e.g., DemoIntegration.log.current).  The log files are stored
        // in <workspace>/../logs
        final Logger log = new ProcessUnsafeLogCreator(PROCESS_NAME, configuration).createLogger(PROCESS_NAME, LogLevel.INFO);

        // All log output should be sent to the Logger we created on the line above
        DelegatingLoggerImpl.setDelegate(log);

        // The Deephaven system uses a wrapper around Java NIO for most network communication and job scheduling, this
        // initializes the communications subsystem with our log
        NIODriver.init(log);

        // The ProcessEnvironment provides hooks for consistent startup and shutdown initialization across processes.
        ProcessEnvironment.basicInteractiveProcessInitialization(configuration, PROCESS_NAME, log);
    }

    /**
     * This static inner class extends ContextAwareRemoteQuery to provide a query class that is specifically
     * designed to execute {@link QueryUtils#getAvgStockPriceTable(Database, String)}
     *
     * This class must also be available to the query worker, so the worker will be able to resolve it when
     * executeQuery is called. This can be done by deploying this class in a jar to the query server
     * or by pushing it to the worker. This example pushes it to the worker. See the rqc.pushClasses
     * call, below.
     */
    public static class GetAveragePrices extends ContextAwareRemoteQuery<Table> {
        private final String date;

        /**
         * Classes that extend ContextAwareRemoteQuery must override execute. This
         * is where the specific query goes that we want to execute. Note that only
         * the Database object is passed in here (it is passed in by the executeQuery
         * method called against the RemoteDatabase). Other parameters must be defined
         * as class variables that are set by the class constructor.
         */
        @Override
        public Table execute(Database db) {
            return QueryUtils.getAvgStockPriceTable(db, date).select();
        }

        /**
         * The class constructor, which takes the parameters needed for this query;
         * these parameter values will be used when execute() is called.
         *
         * @param date Namespace in which to find the table
         */
        public GetAveragePrices(final String date) {
            this.date = date;
        }
    }

    /**
     * Run a sample query to get the average prices of stocks for a given day.
     *
     * @param args 1: The date to use. If not specified, will use currentDateNy.
     */
    public static void main(@NotNull final String... args) {
        // Parse parameter values from the command line
        final String date;
        if (args.length > 0) {
            date = args[0];
        } else {
            date = DBTimeUtils.currentDateNy();
        }

        final long overallStartNanos = System.nanoTime();

        // Set up logging and process management
        prepareEnvironment();

        // Create a Deephaven query client
        final RemoteQueryClient rqc;
        try {
            rqc = new RemoteQueryClient();
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to create RemoteQueryClient", e);
        }

        /*
         * Push the QueryUtils class to the worker. This class will be needed by
         * by the worker when executeQuery() is called.
         */
        try {
            rqc.pushClasses(QueryUtils.class);
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to push QueryUtils class", e);
        }

        // Create a connection to the remote query server specified by RemoteQueryDispatcherParameters.host
        final RemoteDatabase db;
        try {
            db = rqc.getRemoteDB(RemoteProcessingRequest.DEFAULT_MAX_HEAP_SIZE_MB, PROCESS_NAME, RemoteProcessingRequest.DEFAULT_WORKER_DEBUG_ENABLED);
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to create RemoteDatabase", e);
        }

        final long startNanos = System.nanoTime();

        /*
         * Call executeQuery. Within the call, instantiate a new GetAveragePrices query object and pass in
         * the parameters that will be needed to execute the query. The executeQuery method will use the
         * GetAveragePrices class that was pushed through the RemoteQueryClient session to interpret the query object
         * and invoke its execute() method. The execute() method will execute the query and return the resulting Table.
         */
        final Table AvgStockPrices;
        try {
            AvgStockPrices = db.executeQuery(new GetAveragePrices(date));
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to execute remote query.", e);
        }

        final long endNanos = System.nanoTime();

        // Output the results
        System.out.println();
        TableTools.show(AvgStockPrices);
        System.out.println("Query duration " + (endNanos - startNanos) + " ns");
        System.out.println("Overall duration " + (endNanos - overallStartNanos) + " ns");
    }
}

import com.fishlib.configuration.Configuration;
import com.fishlib.io.log.LogLevel;
import com.fishlib.io.logger.DelegatingLoggerImpl;
import com.fishlib.io.logger.Logger;
import com.fishlib.logger.ProcessUnsafeLogCreator;
import com.fishlib.net.impl.nio.NIODriver;
import com.fishlib.util.process.ProcessEnvironment;
import com.illumon.iris.db.tables.Table;
import com.illumon.iris.db.tables.TableDefinition;
import com.illumon.iris.db.tables.databases.Database;
import com.illumon.iris.db.tables.remote.RemoteDatabase;
import com.illumon.iris.db.tables.remotequery.ContextAwareRemoteQuery;
import com.illumon.iris.db.tables.remotequery.RemoteProcessingRequest;
import com.illumon.iris.db.tables.remotequery.RemoteQueryClient;
import com.illumon.iris.db.tables.utils.DBTimeUtils;
import com.illumon.iris.db.tables.utils.TableTools;
import org.jetbrains.annotations.NotNull;

import java.io.IOException;
import java.io.UncheckedIOException;

/**
 * <p>Simple example for executing {@link QueryUtils#getAvgStockPriceTable(Database, String)}
 * against a remote Deephaven database.</p>
 *
 * <p>This example creates a database connection (RemoteDatabase), and uses its methods ad-hoc,
 * comparably to how these methods are used in a Deephaven console session (db.i, db.t, etc).</p>
 *
 * <p>Because the methods are being called against the RemoteDatabase locally in this client application,
 * each operation requires a round-trip between the client and the worker that is associated with the
 * session. For infrequent or very ad-hoc use, this may be fine; for applications where a query
 * will be executed repeatedly, better performance can be achieved by using a ContextAwareRemoteQuery
 * object to allow the method calls to be made as part of a single execution call from the client
 * to the worker. The CARQExample class example performs the same query as this example does,
 * but does it using a ContextAwareRemoteQuery.</p>
 */
public class RemoteDatabaseExample {
    private static final String PROCESS_NAME = "RemoteDatabaseExample";

    private static void prepareEnvironment() {
        // the Configuration object reads the property file, which has named key-value pairs that control the
        // operation of the system.
        final Configuration configuration = Configuration.getInstance();

        // the log object is used by Deephaven components, and produces output in a PROCESS_NAME.log.<date> with
        // a hard link to PROCESS_NAME.log.current (e.g., DemoIntegration.log.current).  The log files are stored
        // in <workspace>/../logs
        final Logger log = new ProcessUnsafeLogCreator(PROCESS_NAME, configuration).createLogger(PROCESS_NAME, LogLevel.INFO);

        // All log output should be sent to the Logger we created on the line above
        DelegatingLoggerImpl.setDelegate(log);

        // The Deephaven system uses a wrapper around Java NIO for most network communication and job scheduling, this
        // initializes the communications subsystem with our log
        NIODriver.init(log);

        // The ProcessEnvironment provides hooks for consistent startup and shutdown initialization across processes.
        ProcessEnvironment.basicInteractiveProcessInitialization(configuration, PROCESS_NAME, log);
    }

    /**
     * Run a sample query to get the average prices of stocks for a given day.
     *
     * @param args 1: The date to use. If not specified, the query will use currentDateNy.
     */
    public static void main(@NotNull final String... args) {
        // Parse parameter values from the command line
        final String date;
        if (args.length > 0) {
            date = args[0];
        } else {
            date = DBTimeUtils.currentDateNy();
        }

        final long overallStartNanos = System.nanoTime();

        // Set up logging and process management
        prepareEnvironment();

        // Create a Deephaven query client
        final RemoteQueryClient rqc;
        try {
            rqc = new RemoteQueryClient();
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to create RemoteQueryClient", e);
        }

        /*
         * Push the QueryUtils class to the worker. This class will be needed by
         * by the worker when executeQuery() is called.
         */
        try {
            rqc.pushClasses(QueryUtils.class);
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to push QueryUtils class", e);
        }

        // Create a connection to the remote query server specified by RemoteQueryDispatcherParameters.host
        final RemoteDatabase db;
        try {
            db = rqc.getRemoteDB(RemoteProcessingRequest.DEFAULT_MAX_HEAP_SIZE_MB, PROCESS_NAME, RemoteProcessingRequest.DEFAULT_WORKER_DEBUG_ENABLED);
        } catch (IOException e) {
            throw new UncheckedIOException("Failed to create RemoteDatabase", e);
        }

        final Table AvgStockPrices = QueryUtils.getAvgStockPriceTable(db, date);

        // Output the results
        TableTools.show(AvgStockPrices);
    }
}