spark-shell --master yarn --com.databricks:spark-avro_2.10:2.0.1
import com.databricks.spark.avro
val jsonDF = “/dir/file.json”
sqlContext.setConf(“spark.sql.avro.compression.codec”, “snappy”)
jsonDF.write.avro("/dest-dir/file.avro")
[ I also tried jsonDF.write.format(“com.databricks.spark.avro”).save("/dest-dir/file.avro") ]
[
It does saves the file as avro but no compression is included can you please guide me on this.
I am working on spark 1.6
]