J'ai un script PySpark vraiment simple qui crée une trame de données à partir de certaines données de parquet sur S3, puis appelle la méthode count()
et imprime le nombre d'enregistrements.
J'exécute le script sur le cluster AWS EMR et je vois les informations étranges suivantes de WARN:
17/12/04 14:20:26 WARN ServletHandler:
javax.servlet.ServletException: Java.util.NoSuchElementException: None.get
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.Java:489)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.Java:427)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.Java:388)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.Java:341)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.Java:228)
at org.spark_project.jetty.servlet.ServletHolder.handle(ServletHolder.Java:845)
at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.Java:1689)
at org.Apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.doFilter(AmIpFilter.Java:164)
at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.Java:1676)
at org.spark_project.jetty.servlet.ServletHandler.doHandle(ServletHandler.Java:581)
at org.spark_project.jetty.server.handler.ContextHandler.doHandle(ContextHandler.Java:1180)
at org.spark_project.jetty.servlet.ServletHandler.doScope(ServletHandler.Java:511)
at org.spark_project.jetty.server.handler.ContextHandler.doScope(ContextHandler.Java:1112)
at org.spark_project.jetty.server.handler.ScopedHandler.handle(ScopedHandler.Java:141)
at org.spark_project.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.Java:461)
at org.spark_project.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.Java:213)
at org.spark_project.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.Java:134)
at org.spark_project.jetty.server.Server.handle(Server.Java:524)
at org.spark_project.jetty.server.HttpChannel.handle(HttpChannel.Java:319)
at org.spark_project.jetty.server.HttpConnection.onFillable(HttpConnection.Java:253)
at org.spark_project.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.Java:273)
at org.spark_project.jetty.io.FillInterest.fillable(FillInterest.Java:95)
at org.spark_project.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.Java:93)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.Java:303)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.Java:148)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.Java:136)
at org.spark_project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.Java:671)
at org.spark_project.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.Java:589)
at Java.lang.Thread.run(Thread.Java:748)
Caused by: Java.util.NoSuchElementException: None.get
at scala.None$.get(Option.scala:347)
at scala.None$.get(Option.scala:345)
at org.Apache.spark.status.api.v1.MetricHelper.submetricQuantiles(AllStagesResource.scala:313)
at org.Apache.spark.status.api.v1.AllStagesResource$$anon$1.build(AllStagesResource.scala:178)
at org.Apache.spark.status.api.v1.AllStagesResource$.taskMetricDistributions(AllStagesResource.scala:181)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$taskSummary$1.apply(OneStageResource.scala:71)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$taskSummary$1.apply(OneStageResource.scala:62)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$withStageAttempt$1.apply(OneStageResource.scala:130)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$withStageAttempt$1.apply(OneStageResource.scala:126)
at org.Apache.spark.status.api.v1.OneStageResource.withStage(OneStageResource.scala:97)
at org.Apache.spark.status.api.v1.OneStageResource.withStageAttempt(OneStageResource.scala:126)
at org.Apache.spark.status.api.v1.OneStageResource.taskSummary(OneStageResource.scala:62)
at Sun.reflect.GeneratedMethodAccessor153.invoke(Unknown Source)
at Sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.Java:43)
at Java.lang.reflect.Method.invoke(Method.Java:498)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.Java:81)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.Java:144)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.Java:161)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.Java:205)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.Java:99)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.Java:389)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.Java:347)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.Java:102)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.Java:326)
at org.glassfish.jersey.internal.Errors$1.call(Errors.Java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.Java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.Java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.Java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.Java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.Java:317)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.Java:305)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.Java:1154)
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.Java:473)
... 28 more
17/12/04 14:20:26 WARN HttpChannel: //ip-172-31-81-10.ec2.internal:4040/api/v1/applications/application_1512395256824_0002/stages/3/0/taskSummary?proxyapproved=true
javax.servlet.ServletException: Java.util.NoSuchElementException: None.get
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.Java:489)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.Java:427)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.Java:388)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.Java:341)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.Java:228)
at org.spark_project.jetty.servlet.ServletHolder.handle(ServletHolder.Java:845)
at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.Java:1689)
at org.Apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.doFilter(AmIpFilter.Java:164)
at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.Java:1676)
at org.spark_project.jetty.servlet.ServletHandler.doHandle(ServletHandler.Java:581)
at org.spark_project.jetty.server.handler.ContextHandler.doHandle(ContextHandler.Java:1180)
at org.spark_project.jetty.servlet.ServletHandler.doScope(ServletHandler.Java:511)
at org.spark_project.jetty.server.handler.ContextHandler.doScope(ContextHandler.Java:1112)
at org.spark_project.jetty.server.handler.ScopedHandler.handle(ScopedHandler.Java:141)
at org.spark_project.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.Java:461)
at org.spark_project.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.Java:213)
at org.spark_project.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.Java:134)
at org.spark_project.jetty.server.Server.handle(Server.Java:524)
at org.spark_project.jetty.server.HttpChannel.handle(HttpChannel.Java:319)
at org.spark_project.jetty.server.HttpConnection.onFillable(HttpConnection.Java:253)
at org.spark_project.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.Java:273)
at org.spark_project.jetty.io.FillInterest.fillable(FillInterest.Java:95)
at org.spark_project.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.Java:93)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.Java:303)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.Java:148)
at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.Java:136)
at org.spark_project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.Java:671)
at org.spark_project.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.Java:589)
at Java.lang.Thread.run(Thread.Java:748)
Caused by: Java.util.NoSuchElementException: None.get
at scala.None$.get(Option.scala:347)
at scala.None$.get(Option.scala:345)
at org.Apache.spark.status.api.v1.MetricHelper.submetricQuantiles(AllStagesResource.scala:313)
at org.Apache.spark.status.api.v1.AllStagesResource$$anon$1.build(AllStagesResource.scala:178)
at org.Apache.spark.status.api.v1.AllStagesResource$.taskMetricDistributions(AllStagesResource.scala:181)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$taskSummary$1.apply(OneStageResource.scala:71)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$taskSummary$1.apply(OneStageResource.scala:62)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$withStageAttempt$1.apply(OneStageResource.scala:130)
at org.Apache.spark.status.api.v1.OneStageResource$$anonfun$withStageAttempt$1.apply(OneStageResource.scala:126)
at org.Apache.spark.status.api.v1.OneStageResource.withStage(OneStageResource.scala:97)
at org.Apache.spark.status.api.v1.OneStageResource.withStageAttempt(OneStageResource.scala:126)
at org.Apache.spark.status.api.v1.OneStageResource.taskSummary(OneStageResource.scala:62)
at Sun.reflect.GeneratedMethodAccessor153.invoke(Unknown Source)
at Sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.Java:43)
at Java.lang.reflect.Method.invoke(Method.Java:498)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.Java:81)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.Java:144)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.Java:161)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.Java:205)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.Java:99)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.Java:389)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.Java:347)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.Java:102)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.Java:326)
at org.glassfish.jersey.internal.Errors$1.call(Errors.Java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.Java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.Java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.Java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.Java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.Java:317)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.Java:305)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.Java:1154)
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.Java:473)
Il semble cependant que cela n'échoue pas. J'ai obtenu le compte avec succès.
Demandez-vous simplement si quelqu'un sait pourquoi cela se produit et comment s'en débarrasser.
Merci
Ces messages d'avertissement peuvent être supprimés en ajoutant les lignes suivantes à /etc/spark/conf/log4j.properties
:
log4j.logger.org.spark_project.jetty.server.HttpChannel=ERROR
log4j.logger.org.spark_project.jetty.servlet.ServletHandler=ERROR
Je n'ai vu aucun impact sur les performances ni sur la stabilité de l'emploi. Maintenant, mes journaux sont bien lisibles :)
Pour ceux qui utilisent AWS EMR avec Spark, j'ai également reçu le même message d'erreur lors de l'exécution de mes travaux Spark dans
Rétrogradation simple vers emr-5.9.0
vous aidera à résoudre ce problème.
J'espère que cela aide.