2013-11-22 69 views
1

请帮忙。我在网上搜索,找不到任何东西。大部分类似的问题都没有答案或无益。运行Hadoop示例Jar

嗨我试图运行Hadoop Pi的例子。我的设置完成并成功。我运行bim/hadoop dfs -ls,但没有出错。

但这:

Sanjanas-MacBook-Pro:hadoop sanjanaagarwal$ /usr/local/Hadoop/bin/hadoop jar $HADOOP_HOME/hadoop-examples-*.jar pi 10 100 
Number of Maps = 10 
Samples per Map = 100 
13/11/21 20:57:47 WARN hdfs.DFSClient: DataStreamer Exception: org.apache.hadoop.ipc.RemoteException: java.io.IOException: File /user/sanjanaagarwal/PiEstimator_TMP_3_141592654/in/part0 could only be replicated to 0 nodes, instead of 1 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1591) 
    at org.apache.hadoop.hdfs.server.namenode.NameNode.addBlock(NameNode.java:771) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:557) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1439) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1435) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:394) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1278) 
    at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1433) 

    at org.apache.hadoop.ipc.Client.call(Client.java:1150) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:226) 
    at com.sun.proxy.$Proxy0.addBlock(Unknown Source) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82) 
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59) 
    at com.sun.proxy.$Proxy0.addBlock(Unknown Source) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:3719) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:3586) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$2400(DFSClient.java:2792) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2987) 

13/11/21 20:57:47 WARN hdfs.DFSClient: Error Recovery for block null bad datanode[0] nodes == null 
13/11/21 20:57:47 WARN hdfs.DFSClient: Could not get block locations. Source file "/user/sanjanaagarwal/PiEstimator_TMP_3_141592654/in/part0" - Aborting... 
org.apache.hadoop.ipc.RemoteException: java.io.IOException: File /user/sanjanaagarwal/PiEstimator_TMP_3_141592654/in/part0 could only be replicated to 0 nodes, instead of 1 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1591) 
    at org.apache.hadoop.hdfs.server.namenode.NameNode.addBlock(NameNode.java:771) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:557) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1439) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1435) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:394) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1278) 
    at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1433) 

    at org.apache.hadoop.ipc.Client.call(Client.java:1150) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:226) 
    at com.sun.proxy.$Proxy0.addBlock(Unknown Source) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82) 
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59) 
    at com.sun.proxy.$Proxy0.addBlock(Unknown Source) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:3719) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:3586) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$2400(DFSClient.java:2792) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2987) 
13/11/21 20:57:47 ERROR hdfs.DFSClient: Exception closing file /user/sanjanaagarwal/PiEstimator_TMP_3_141592654/in/part0 : org.apache.hadoop.ipc.RemoteException: java.io.IOException: File /user/sanjanaagarwal/PiEstimator_TMP_3_141592654/in/part0 could only be replicated to 0 nodes, instead of 1 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1591) 
    at org.apache.hadoop.hdfs.server.namenode.NameNode.addBlock(NameNode.java:771) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:557) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1439) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1435) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:394) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1278) 
    at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1433) 

org.apache.hadoop.ipc.RemoteException: java.io.IOException: File /user/sanjanaagarwal/PiEstimator_TMP_3_141592654/in/part0 could only be replicated to 0 nodes, instead of 1 
    at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:1591) 
    at org.apache.hadoop.hdfs.server.namenode.NameNode.addBlock(NameNode.java:771) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:557) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1439) 
    at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:1435) 
    at java.security.AccessController.doPrivileged(Native Method) 
    at javax.security.auth.Subject.doAs(Subject.java:394) 
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1278) 
    at org.apache.hadoop.ipc.Server$Handler.run(Server.java:1433) 

    at org.apache.hadoop.ipc.Client.call(Client.java:1150) 
    at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:226) 
    at com.sun.proxy.$Proxy0.addBlock(Unknown Source) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) 
    at java.lang.reflect.Method.invoke(Method.java:597) 
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:82) 
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:59) 
    at com.sun.proxy.$Proxy0.addBlock(Unknown Source) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.locateFollowingBlock(DFSClient.java:3719) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.nextBlockOutputStream(DFSClient.java:3586) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$2400(DFSClient.java:2792) 
    at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2987) 

我hadoop-env.sh是所有设置。仍然不知道这里的问题是什么..请提及命令,如果有什么要检查,因为我对Linux非常新。谢谢

+0

你可以粘贴到你的问题运行'hadoop fsck /' –

+0

@HenryShen输出正如我在问题中提到,我已经做了很多在线搜索,没有任何帮助。 –

回答

0

无法解决错误,所以找到了一个解决方法,我从virtualbox上的云时代获得了一个虚拟机,并在其上运行了cloudera。

2

我认为你的集群的DataNode没有运行。使用jps命令进行检查。

+0

你是对的。我已经这样做了。但不知道为什么它没有运行 –

+0

只需删除DataNode中的dfs文件夹并再次启动群集。如果仍然没有运行,那么你的配置文件可能有问题。检查/ etc/hosts文件和其他文件中的条目。 –

+0

试过了。最后从cloudera得到了一个虚拟机,然后在ubuntu上运行 –