星火1.3.1 SQL libriary：scala.reflect.internal.MissingRequirementError：类org.apache.spark.sql.catalyst.ScalaReflection

Question

当尝试运行这段代码：

val conf = new SparkConf() 
    .setMaster("local[1]") 
    .setAppName("Small") 
    .set("spark.executor.memory", "2g") 
val sc = new SparkContext(conf) 

val sqlContext = new org.apache.spark.sql.SQLContext(sc) 
import sqlContext.implicits._ 
val df = sc.parallelize(Array((1,30),(2,10),(3,20),(1,10)(2,30))).toDF("books","readers") 
val results = df.join(
df.select($"books" as "r_books", $"readers" as "r_readers"), 
$"readers" === $"r_readers" and $"books" < $"r_books" 
) 
.groupBy($"books", $"r_books") 
.agg($"books", $"r_books", count($"readers")) 

在SBT控制台开始与以下build.sbt：

name := "Small" 

version := "1.0" 

scalaVersion := "2.10.4" 

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.3.1" 

libraryDependencies += "org.apache.spark" %% "spark-sql" % "1.3.1" 

返回错误：

scala.reflect.internal.MissingRequirementError：类org.apache.spark.sql.catalyst.ScalaReflection在JavaMirror与java.net.URLClass Loader @ 13a9a4f9 ...

任何想法？

Answer 1

你有一些拼写错误，最后的最后一个函数计数不能有您需要导入org.apache.spark.sql.functions._

检查这一点，我CORRCT拼写错误，并增加进口

import org.apache.spark.{ SparkContext, SparkConf } 
import org.apache.spark.sql.functions._ 

/** 
* Created by anquegi on 01/06/15. 
*/ 
object QSpark162015 extends App { 

    val conf = new SparkConf() 
    .setMaster("local[2]") 
    .setAppName("QSpark162015") 
    .set("spark.executor.memory", "2g") 

    val sc = new SparkContext(conf) 

    val sqlContext = new org.apache.spark.sql.SQLContext(sc) 

    import sqlContext.implicits._ 

    val df = sc.parallelize(Array((1, 30), (2, 10), (3, 20), (1, 10), (2, 30))).toDF("books", "readers") 
    val results = df.join(
    df.select($"books" as "r_books", $"readers" as "r_readers"), 
    $"readers" === $"r_readers" and $"books" < $"r_books" 
) 
    .groupBy($"books", $"r_books") 
    .agg($"books", $"r_books", count($"readers")) 

    results.foreach(println _) 

    sc.stop() 

}