Spark 2.0 - 将DataFrame转换为DataSet

Question

我想加载我的数据并对其进行一些基本的线性回归。所以首先，我需要使用VectorAssembler来制作我的功能栏。但是，当我使用assembler.transform(df)时，df是DataFrame，并且它期望DataSet。我试过df.toDS，但它给value toDS is not a member of org.apache.spark.sql.DataFrame。的确，它是org.apache.spark.sql.DatasetHolder的成员。

我在这里发生了什么问题？

package main.scala 

import org.apache.spark.SparkContext 
import org.apache.spark.SparkContext._ 
import org.apache.spark.SparkConf 
import org.apache.spark.sql.functions._ 
import org.apache.spark.sql.SQLContext 
import org.apache.spark.sql.DatasetHolder 
import org.apache.spark.ml.regression.LinearRegression 
import org.apache.spark.ml.feature.RFormula 
import org.apache.spark.ml.feature.VectorAssembler 
import org.apache.spark.ml.linalg.Vectors 

object Analyzer { 
    def main(args: Array[String]) { 

    val conf = new SparkConf() 
    val sc = new SparkContext(conf) 
    val sqlContext = new SQLContext(sc)  
    import sqlContext.implicits._ 

    val df = sqlContext.read 
    .format("com.databricks.spark.csv") 
    .option("header", "false") 
    .option("delimiter", "\t") 
    .option("parserLib", "UNIVOCITY") 
    .option("inferSchema", "true") 
    .load("data/snap/*") 

    val assembler = new VectorAssembler() 
    .setInputCols(Array("own", "want", "wish", "trade", "comment")) 
    .setOutputCol("features") 

    val df1 = assembler.transform(df) 

    val formula = new RFormula().setFormula("rank ~ own + want + wish + trade + comment") 
    .setFeaturesCol("features") 
     .setLabelCol("rank") 
} 
} 

Answer 1

显然的问题是，因为我仍在使用Spark 1.6风格的Spark和SQLContext。我更改为SparkSession，而transform()能够隐式接受DataFrame。

package main.scala 

import org.apache.spark.sql.SparkSession 
import org.apache.spark.sql.Dataset 
import org.apache.spark.ml.regression.LinearRegression 
import org.apache.spark.ml.feature.RFormula 
import org.apache.spark.ml.feature.VectorAssembler 
import org.apache.spark.ml.linalg.Vectors 

object Analyzer { 
    def main(args: Array[String]) { 

     val spark = SparkSession.builder().getOrCreate() 
     import spark.implicits._ 

     val df = spark.read 
     .format("com.databricks.spark.csv") 
     .option("header", "false") 
     .option("delimiter", "\t") 
     .option("parserLib", "UNIVOCITY") 
     .option("inferSchema", "true") 
     .load("data/snap/*")   

     df.show() 

     val assembler = new VectorAssembler() 
     .setInputCols(Array("own", "want", "wish", "trade", "comment")) 
     .setOutputCol("features") 

     val df1 = assembler.transform(df) 
    } 
}