线程“main”中的异常java.lang.NoClassDefFoundError：org/apache/spark/internal/Logging

My Spark Consumer失败并显示“logging”错误。我发现浏览错误是由于罐子不兼容。我使用的Spark 1.6.3和所有的依赖关系在POM中使用，XML是1.6.3。我仍然得到同样的错误。下面是我的maven结构和执行时抛出错误。线程“main”中的异常java.lang.NoClassDefFoundError：org/apache/spark/internal/Logging

Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/spark/internal/Logging 
    at java.lang.ClassLoader.defineClass1(Native Method) 
    at java.lang.ClassLoader.defineClass(ClassLoader.java:763) 
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) 
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) 
    at java.net.URLClassLoader.access$100(URLClassLoader.java:73) 
    at java.net.URLClassLoader$1.run(URLClassLoader.java:368) 
    at java.net.URLClassLoader$1.run(URLClassLoader.java:362) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at java.net.URLClassLoader.findClass(URLClassLoader.java:361) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424) 
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357) 
    at org.apache.spark.streaming.kafka.KafkaUtils$.createStream(KafkaUtils.scala:91) 
    at org.apache.spark.streaming.kafka.KafkaUtils$.createStream(KafkaUtils.scala:66) 
    at org.apache.spark.streaming.kafka.KafkaUtils$.createStream(KafkaUtils.scala:130) 
    at org.apache.spark.streaming.kafka.KafkaUtils.createStream(KafkaUtils.scala) 
    at datapipeline.SparkConsumer.main(SparkConsumer.java:60) 
Caused by: java.lang.ClassNotFoundException: org.apache.spark.internal.Logging 
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381) 
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424) 
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)

我的pom.xml：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 
    <modelVersion>4.0.0</modelVersion> 
    <groupId>com.iot.app.kafka</groupId> 
    <artifactId>iotanalytics</artifactId> 
    <!-- <packaging>jar</packaging> --> 
    <version>1.0.0</version> 
    <name>IoT Kafka Producer</name> 


    <dependencies> 
     <dependency> 
      <groupId>org.apache.kafka</groupId> 
      <artifactId>kafka-clients</artifactId> 
      <version>0.10.0.0</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-network-common_2.10 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-network-common_2.10</artifactId> 
      <version>1.6.3</version> 
     </dependency> 



     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-10_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming-kafka-0-10_2.11</artifactId> 
      <version>2.0.0</version> 
     </dependency> 



     <dependency> 
      <groupId>org.apache.avro</groupId> 
      <artifactId>avro</artifactId> 
      <version>1.8.0</version> 
     </dependency> 

     <dependency> 
      <groupId>com.fasterxml.jackson.core</groupId> 
      <artifactId>jackson-databind</artifactId> 
      <version>2.8.2</version> 
     </dependency> 

     <dependency> 
     <groupId>com.fasterxml.jackson.module</groupId> 
     <artifactId>jackson-module-scala_2.10</artifactId> 
     <version>2.8.2</version> 
     </dependency> 

     <dependency> 
      <groupId>com.fasterxml.jackson.core</groupId> 
      <artifactId>jackson-annotations</artifactId> 
      <version>2.8.2</version> 
     </dependency> 

     <dependency> 
      <groupId>com.fasterxml.jackson.core</groupId> 
      <artifactId>jackson-core</artifactId> 
      <version>2.8.2</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/javax.json/javax.json-api --> 
     <dependency> 
      <groupId>javax.json</groupId> 
      <artifactId>javax.json-api</artifactId> 
      <version>1.1</version> 
     </dependency> 



     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming_2.11 --> 

     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming_2.11</artifactId> 
      <version>1.6.3</version> 
      <scope>provided</scope> 
     </dependency> 

     <dependency> 
      <groupId>org.glassfish</groupId> 
      <artifactId>javax.json</artifactId> 
      <version>1.0.4</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client --> 
     <dependency> 
      <groupId>org.apache.hadoop</groupId> 
      <artifactId>hadoop-client</artifactId> 
      <version>2.2.0</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-core_2.11</artifactId> 
      <version>1.6.3</version> 
      <scope>provided</scope> 
     </dependency> 




     <dependency> 
      <groupId>org.spark-project.spark</groupId> 
      <artifactId>unused</artifactId> 
      <version>1.0.0</version> 
      <scope>provided</scope> 
     </dependency> 




     <dependency> 
      <groupId>log4j</groupId> 
      <artifactId>log4j</artifactId> 
      <version>1.2.17</version> 
     </dependency> 


     <!-- <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> 
      <version>2.11.8</version> </dependency> --> 


     <!-- https://mvnrepository.com/artifact/org.scala-lang/scala-library --> 
     <dependency> 
      <groupId>org.scala-lang</groupId> 
      <artifactId>scala-library</artifactId> 
      <version>2.11.9</version> 
     </dependency> 


     <dependency> 
      <groupId>com.typesafe.akka</groupId> 
      <artifactId>akka-actor_2.11</artifactId> 
      <version>2.3.11</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka-0-8-assembly_2.11 --> 
<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-streaming-kafka-0-8-assembly_2.11</artifactId> 
    <version>2.0.0</version> 
</dependency> 



     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-sql_2.10</artifactId> 
      <version>1.6.3</version> 
     </dependency> 

     <dependency> 
      <groupId>junit</groupId> 
      <artifactId>junit</artifactId> 
      <version>4.4</version> 
      <scope>test</scope> 
     </dependency> 



    </dependencies> 


    <properties> 
     <maven.compiler.source>1.8</maven.compiler.source> 
     <maven.compiler.target>1.8</maven.compiler.target> 

    </properties> 

</project>

来源

2017-08-11 user3837415

你有没有试过保存你的pom.xml，右键单击你的项目，mavan构建？也许它只是没有更新。 – Nathan

您可以简单地按Ctrl + Shift + T并搜索“Logging”类，如果您可以看到多个相同的版本，则表示它正在加载您需要的旧一个依赖关系[称为传递maven依赖关系]通过排除它来解决。 –

@MonisMajeed我也尝试通过删除库中的所有jar来构建新的项目。仍然没有运气。我 – user3837415

-1

类org.apache.spark.Logging是Spark版本1.5.2可用或更低不在更高版本。

尝试降级星火版本1.5.2

<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.11 --> 
<dependency> 
    <groupId>org.apache.spark</groupId> 
    <artifactId>spark-core_2.11</artifactId> 
    <version>1.5.2</version> 
    <scope>provided</scope> 
</dependency>

来源

2017-08-11 09:12:10

出于好奇：Spark日志的更新版本如何？我无法想象他们放弃了伐木？换句话说：不应该有这个库的替代品与OP所使用的新版火花版本匹配吗？这听起来是错误的降级整个事情，以使其工作... – GhostCat

你的意思是org.apache.spark.internal.Logging？ – toongeorges

日志记录总是存在于Spark中，你们可以将其作为依赖检查https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 –

有目前的火花的低版本的，我以前BUIL罐子。那些没有被刷新。我删除了旧版本的jar并检查了我的pom.xml并确保所有的依赖关系具有相同的版本并构建了项目。程序成功运行。这里是我的pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 
    <modelVersion>4.0.0</modelVersion> 
    <groupId>com.iot.app.kafka</groupId> 
    <artifactId>DeviceStreaming</artifactId> 
    <packaging>jar</packaging> 
    <version>1.0.0</version> 
    <name>DeviceStreaming</name> 


    <dependencies> 

     <dependency> 
      <groupId>org.apache.kafka</groupId> 
      <artifactId>kafka-clients</artifactId> 
      <version>0.9.0.0</version> 
     </dependency> 

     <!-- <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-network-common_2.11</artifactId> 
      <version>2.1.0</version> </dependency> --> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-network-common_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-network-common_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming-kafka_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming-kafka_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 





     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming_2.11 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.10 --> 
     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-sql_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.glassfish/javax.json --> 
     <dependency> 
      <groupId>org.glassfish</groupId> 
      <artifactId>javax.json</artifactId> 
      <version>1.0.4</version> 
     </dependency> 


     <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client --> 
     <dependency> 
      <groupId>org.apache.hadoop</groupId> 
      <artifactId>hadoop-client</artifactId> 
      <version>2.2.0</version> 
     </dependency> 


     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-core_2.11</artifactId> 
      <version>1.6.3</version> 
     </dependency> 


     <dependency> 
      <groupId>org.spark-project.spark</groupId> 
      <artifactId>unused</artifactId> 
      <version>1.0.0</version> 
      <scope>provided</scope> 
     </dependency> 



     <!-- https://mvnrepository.com/artifact/javax.json/javax.json-api --> 
     <dependency> 
      <groupId>javax.json</groupId> 
      <artifactId>javax.json-api</artifactId> 
      <version>1.1</version> 
     </dependency> 








     <dependency> 
      <groupId>log4j</groupId> 
      <artifactId>log4j</artifactId> 
      <version>1.2.17</version> 
     </dependency> 
     <dependency> 
      <groupId>junit</groupId> 
      <artifactId>junit</artifactId> 
      <version>4.12</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/com.google.guava/guava --> 
     <dependency> 
      <groupId>com.google.guava</groupId> 
      <artifactId>guava</artifactId> 
      <version>14.0.1</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/org.onosproject/onos-api --> 
     <dependency> 
      <groupId>org.onosproject</groupId> 
      <artifactId>onos-api</artifactId> 
      <version>1.6.0</version> 
     </dependency> 



     <dependency> 
      <groupId>org.scala-lang</groupId> 
      <artifactId>scala-library</artifactId> 
      <version>2.11.0</version> 
     </dependency> 




     <dependency> 
      <groupId>org.apache.spark</groupId> 
      <artifactId>spark-streaming-kafka-0-8-assembly_2.11</artifactId> 
      <version>2.0.0</version> 
     </dependency> 

     <dependency> 
      <groupId>org.apache.avro</groupId> 
      <artifactId>avro</artifactId> 
      <version>1.8.0</version> 
     </dependency> 

     <dependency> 
      <groupId>com.twitter</groupId> 
      <artifactId>bijection-avro_2.10</artifactId> 
      <version>0.9.2</version> 
     </dependency> 

     <!-- https://mvnrepository.com/artifact/com.googlecode.json-simple/json-simple --> 
<dependency> 
    <groupId>com.googlecode.json-simple</groupId> 
    <artifactId>json-simple</artifactId> 
    <version>1.1</version> 
</dependency> 


    </dependencies> 


    <properties> 
     <maven.compiler.source>1.8</maven.compiler.source> 
     <maven.compiler.target>1.8</maven.compiler.target> 

    </properties> 


</project>

来源

2017-08-22 13:46:32 user3837415

线程“main”中的异常java.lang.NoClassDefFoundError：org/apache/spark/internal/Logging

回答

相关问题