- I’m getting below error when MyDocument.parseFrom is called (see my code snippet)
- Each Row has two columns; 1) column is a string 2) is byte array (protobuf array)
- I’m trying to past 2nd column value to parseFrom method
Any help is really appreciated.
Below is the error
[info] Compiling 1 protobuf files to C:\scala\target\scala-2.12\src_managed\main\scalapb
[info] compiling 27 Scala sources to C:\scala\target\scala-2.12\classes ...
[error] C:\scala\MyConnector.scala:79:33: overloaded method value parseFrom with alternatives:
[error] (_input__: com.google.protobuf.CodedInputStream)grpc.scala.v1.MyEvents.MyDocument <and>
[error] (s: Array[Byte])grpc.scala.v1.MyEvents.PatientDocument <and>
[error] (input: java.io.InputStream)grpc.scala.v1.MyEvents.MyDocument
[error] cannot be applied to (Any)
[error] val p = MyDocument.parseFrom(r(1))
[error] ^
[error] one error found
[error] (Compile / compileIncremental) Compilation failed
[error] Total time: 13 s, completed May 17, 2022, 4:07:00 PM
My Code Snippet
val sqlContext = new org.apache.spark.sql.SQLContext(spark.sparkContext)
df.createOrReplaceTempView("my_documents")
val sample_doc_df = sqlContext.sql("SELECT * FROM my_documents")
for (r <- sample_doc_df.collect()) {
val p = MyDocument.parseFrom(r(1))
println(s"id = ${p.Id}, HCS = ${p.depatment}")
}
build.bat file
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / scalaVersion := "2.12.15"
ThisBuild / assemblyMergeStrategy := {
case PathList("META-INF", "MANIFEST.MF") => MergeStrategy.discard
case PathList("META-INF", xs @ _*) => MergeStrategy.last
case x => MergeStrategy.first
}
lazy val root = project
.in(file("."))
.settings(
assembly / assemblyJarName := "com-self-conector_2.12-0.1.0-SNAPSHOT.jar",
libraryDependencies += "com.typesafe" % "config" % "1.4.2",
libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.2.1" % "provided",
libraryDependencies += "com.github.scopt" %% "scopt" % "3.6.0",
libraryDependencies += "com.thesamet.scalapb" %% "sparksql-scalapb" % "0.11.0" % "provided",
libraryDependencies += "com.google.protobuf" % "protobuf-java" % "3.13.0",
)
Compile / PB.targets := Seq(
scalapb.gen() -> (Compile / sourceManaged).value / "scalapb"
)