* * Example1 (scenario and configuration are loaded from the file): * * Scenario: * * {@code * * test * * some source connection * select from some_table * * * } * * Configuration file: * * {@code * * * * org.apache.derby.jdbc.EmbeddedDriver * jdbc:derby:/demo/javadb * * * some_folder/*.json * com.toolsverse.etl.connector.json.JsonConnector * * * * * * * * * * * } * * * Example2 (scenario and configuration created programmatically): * * {@code * * Alias source = new Alias(); * source.setName("Java DB"); * source.setUrl("jdbc:derby:some_folder/demo/javadb"); * source.setJdbcDriverClass("org.apache.derby.jdbc.EmbeddedDriver"); * * config.addAliasToMap("some source connection", source); * * Source source1 = new Source(); * source1.setName("test"); * source1.setSql("select * from some_table"); * source1.setConnectionName("some source connection"); * scenario.addSource(source1); * } *
* If the are multiple destination connection you can configure ETL scenario to load data in parallel threads, one thread per each * destination connection. When data for all connections processed the ETL engine will issue a coordinated commit or rollback. * *
* * Example3 (scenario is loaded from the file): * * {@code * * ..... * * } * * Example4 (scenario is created programmatically): * * {@code * Scenario scenario = new Scenario(); * scenario.setParallelConnections(true); * ..... * * } *
* Data are streamed from the source database to the destination which means the memory consumption is minimal. *
* There are no input files. Output files are: APP_HOME/data/EMPLOYEE.json, APP_HOME/data/CONFIG_PROPERTY.json and * APP_HOME/data/CONFIG_VALUE.json *
* The logger is manually set to the INFO mode which increases verbosity of the etl engine. * */ public class UsingNamedConnections { /** * Configures ETL engine and executes ETL scenario. * * @param args the command line arguments */ public static void main(String[] args) { UsingNamedConnections engine = new UsingNamedConnections(); try { // instantiates ETL configuration EtlConfig etlConfig = new EtlConfig(); // creates embedded ETL process EtlProcess etlProcess = new EtlProcess(EtlProcess.EtlMode.EMBEDDED); // print out framework version System.out.println(SystemConfig.instance().getTitle( EtlConfig.DEFAULT_TITLE) + " " + SystemConfig.instance().getSystemProperty( SystemConfig.VERSION)); // creates configuration which contains source and destination // connections and ETL scenario name, loads and executes ETL // scenario. EtlResponse response = engine.createConfigAndExecute(etlConfig, "Examples/Engine/db2filenamedconnections.xml", etlProcess); // print out formatted output from the ETL response System.out.println(engine.getMessage(response, "Examples/Engine/db2filenamedconnections.xml")); } catch (Exception ex) { System.out.println(Utils.getStackTraceAsString(ex)); } System.exit(0); } /** * Manually creates connection aliases for the ETL process, loads given ETL scenario from the default folder. * Executes ETl scenario. * @param config the ETL config * @param scenariFileName the ETL scenario file name * @param etlProcess the ETl process * @return ETL response * @throws Exception in case of any error */ private EtlResponse createConfigAndExecute(EtlConfig config, String scenariFileName, EtlProcess etlProcess) throws Exception { // initializes ETl config config.init(); // creates source alias Alias source = new Alias(); source.setName("Java DB"); source.setUrl("jdbc:derby:{app.root.data}/demo/javadb"); source.setJdbcDriverClass("org.apache.derby.jdbc.EmbeddedDriver"); // creates destination alias Alias destination = new Alias(); destination.setName("JSON files"); destination .setConnectorClassName("com.toolsverse.etl.connector.json.JsonConnector"); destination.setUrl("{app.root.data}/*.json"); // adds aliases. ETL process will create connections from these aliases config.addAliasToMap("the source connection", source); config.addAliasToMap("the destination connection", destination); // instantiates ETL factory EtlFactory etlFactory = new EtlFactory(); // loads given ETl scenario, sets ETL action Scenario scenario = etlFactory.getScenario(config, scenariFileName); if (scenario == null) { System.exit(0); } scenario.setAction(EtlConfig.EXTRACT_LOAD); // creates ETl request using given config, scenario and log level EtlRequest request = new EtlRequest(config, scenario, Logger.INFO); // executes ETL process return etlProcess.execute(request); } /** * Creates the formatted message from the ETL response * * @param response the ETl response * @param scenarioName the ETL scenario name * * @return formatted message from the ETL response */ private String getMessage(EtlResponse response, String scenarioName) { String msg = ""; String start = response != null ? response.getStartTime().toString() : new Date().toString(); String end = response != null && response.getEndTime() != null ? response .getEndTime().toString() : new Date().toString(); String diff = response != null ? String .valueOf(Utils.getDateDiff(response.getEndTime(), response.getStartTime(), Calendar.SECOND)) : "0"; if (response.getRetCode() == EtlConfig.RETURN_OK) { msg = Utils .format("\n" + EtlResource.FINISH_SUCCESS_MSG.getValue() + "\n" + "Scenario name: %1" + "\n" + "Started at %2, finished at %3, total execution time %4 seconds.", scenarioName, start, end, diff); if (!Utils.isNothing(response.getWarnings())) { msg = msg + "\n" + EtlResource.WARING_NOTHING_WAS_EXECUTED.getValue(); } } else if (response.getRetCode() == EtlConfig.RETURN_NO_DRIVERS) { msg = "\n" + EtlResource.NO_DRIVERS_MSG.getValue(); } else { msg = Utils .format("\n" + EtlResource.FINISH_ERROR_MSG.getValue() + "\n" + "Scenario name: %1" + "\n" + "Started at %2, finished at %3, total execution time %4 seconds.", scenarioName, start, end, diff) + "\n" + Utils.getStackTraceAsString(response.getException()); } return msg; } }