Java中使用嵌入式Cassandra服务器测试Cassandra-Spark作业的示例

Question

我是Cassandra和Spark的新手。我想建立一个测试我的星火工作，其执行以下操作：从表一

1. 数据装载到DataFrames
2. 做一些过滤，分组和汇总这些DataFrames
3. 加载结果到表B

我想使用嵌入式Cassandra服务器来运行测试，而不是让它连接到Cassandra数据库的本地实例。有没有人做过这个？如果是这样，请有人指点我一个很好的例子吗？感谢您的帮助！

Answer 1

this code does 

package cassspark.clt; 

import java.io.*; 
import javafx.application.Application; 
import java.util.concurrent.Executors ; 
import java.util.concurrent.ExecutorService; 
import org.apache.cassandra.service.CassandraDaemon; 
import com.datastax.driver.core.exceptions.ConnectionException; 
import java.util.Properties; 
import org.apache.log4j.PropertyConfigurator; 
import org.apache.spark.sql.SparkSession; 

public class EmbeddedCassandraDemo extends Application { 

    private ExecutorService executor = Executors.newSingleThreadExecutor(); 
    private CassandraDaemon cassandraDaemon; 

    public EmbeddedCassandraDemo() { 
    } 

    public static void main(String[] args) { 
     try { 
      new EmbeddedCassandraDemo().run(); 
     } 
     catch(java.lang.InterruptedException e) 
     { 
      ; 
     } 
    } 

    @Override public void start(javafx.stage.Stage stage) throws Exception 
    { 
     stage.show(); 
    } 

    private void run() throws InterruptedException, ConnectionException { 
     setProperties(); 
     activateDeamon(); 
    } 

    private void activateDeamon() { 
     executor.execute(new Runnable() { 

      @Override 
      public void run() { 
       cassandraDaemon = new CassandraDaemon(); 
       cassandraDaemon.activate(); 
       SparkSession spark = SparkSession .builder().master("local").appName("ASH").getOrCreate(); 
      } 
     }); 
    } 

    private void setProperties() { 

     final String yaml = System.getProperty("user.dir") + File.separator +"conf"+File.separator+"cassandra.yaml"; 
     final String storage = System.getProperty("user.dir") + File.separator +"storage" + File.separator +"data"; 

     System.setProperty("cassandra.config", "file:"+ yaml); 
     System.setProperty("cassandra.storagedir", storage); 
     System.setProperty("cassandra-foreground", "true"); 

     String log4JPropertyFile = "./conf/log4j.properties"; 
     Properties p = new Properties(); 
     try { 
      p.load(new FileInputStream(log4JPropertyFile)); 
      PropertyConfigurator.configure(p); 
     } catch (IOException e) { 
      System.err.println("./conf/log4j.properties not found "); 
      System.exit(1); 
      ; 
     } 
    } 
}