i new cassandra , spark. trying set test spark job, following:
- loads data table dataframes
- does filtering, grouping , aggregating on these dataframes
- loads result table b
i want use embedded cassandra server run test rather having connecting local instance of cassandra database. has done before? if so, point me example please? in advance!
this code package cassspark.clt; import java.io.*; import javafx.application.application; import java.util.concurrent.executors ; import java.util.concurrent.executorservice; import org.apache.cassandra.service.cassandradaemon; import com.datastax.driver.core.exceptions.connectionexception; import java.util.properties; import org.apache.log4j.propertyconfigurator; import org.apache.spark.sql.sparksession; public class embeddedcassandrademo extends application { private executorservice executor = executors.newsinglethreadexecutor(); private cassandradaemon cassandradaemon; public embeddedcassandrademo() { } public static void main(string[] args) { try { new embeddedcassandrademo().run(); } catch(java.lang.interruptedexception e) { ; } } @override public void start(javafx.stage.stage stage) throws exception { stage.show(); } private void run() throws interruptedexception, connectionexception { setproperties(); activatedeamon(); } private void activatedeamon() { executor.execute( new runnable() { @override public void run() { cassandradaemon = new cassandradaemon(); cassandradaemon.activate(); sparksession spark = sparksession .builder().master("local").appname("ash").getorcreate(); } }); } private void setproperties() { final string yaml = system.getproperty("user.dir") + file.separator +"conf"+file.separator+"cassandra.yaml"; final string storage = system.getproperty("user.dir") + file.separator +"storage" + file.separator +"data"; system.setproperty("cassandra.config", "file:"+ yaml ); system.setproperty("cassandra.storagedir", storage ); system.setproperty("cassandra-foreground", "true"); string log4jpropertyfile = "./conf/log4j.properties"; properties p = new properties(); try { p.load(new fileinputstream(log4jpropertyfile)); propertyconfigurator.configure(p); } catch (ioexception e) { system.err.println("./conf/log4j.properties not found "); system.exit(1); ; } } }
Comments
Post a Comment