@shyamspr
Thanks for your response. The file has only 1 row of data and I'm using the community edition of databricks. Also please see attached the parquet file that was generated. Please note that I used Parquet.Net (.Net Library for generating parquet File) Nuget Package to generate Parquet File. Also I uploaded the parquet file manually to DBFS. Is there any way I can upload the parquet file so you can validate and let me know where the issue is?
var itemIDColumn = new Parquet.Data.DataColumn(new DataField<string>("ItemID"), new string[] { consumersRatingInfo.ItemID });
var retailerIDColumn = new Parquet.Data.DataColumn(new DataField<string>("RetailerID"), new string[] { consumersRatingInfo.RetailerID });
//var ratingDateTimeColumn = new Parquet.Data.DataColumn(new DateTimeDataField<DateTime>("RatingDateTime"), new DateTime[] { consumersRatingInfo.RatingDateTime });
//var ratingDateTimeColumn = new Parquet.Data.DataColumn(new DateTimeDataField("dateDateAndTime", DateTimeFormat.DateAndTime), new DateTimeOffset[] { consumersRatingInfo.RatingDateTime });
var ratingValueColumn = new Parquet.Data.DataColumn(new DataField<int>("RatingValue"), new int[] { consumersRatingInfo.RatingValue });
var itemUPCColumn = new Parquet.Data.DataColumn(new DataField<string>("ItemUPC"), new string[] { consumersRatingInfo.ItemUPC });
var greceiptIDColumn = new Parquet.Data.DataColumn(new DataField<string>("GReceiptID"), new string[] { consumersRatingInfo.GReceiptID });
var itemNameColumn = new Parquet.Data.DataColumn(new DataField<string>("ItemName"), new string[] { consumersRatingInfo.ItemName });
var ratingLabelColumn = new Parquet.Data.DataColumn(new DataField<string>("RatingLabel"), new string[] { consumersRatingInfo.RatingLabel });
var retailerTypeColumn = new Parquet.Data.DataColumn(new DataField<string>("RetailerType"), new string[] { consumersRatingInfo.RetailerType });
//var ratingColumn = new Parquet.Data.DataColumn(new DataField<List<ConsumersRatingAttributesInfo>>("Rating"), new List<ConsumersRatingAttributesInfo>[] { consumersRatingAttributesInfos });
// create file schema
var schema = new Schema(idColumn.Field, cityColumn.Field, itemIDColumn.Field, retailerIDColumn.Field, ratingValueColumn.Field, itemUPCColumn.Field, greceiptIDColumn.Field, itemNameColumn.Field, ratingLabelColumn.Field, retailerTypeColumn.Field);
using (Stream fileStream = System.IO.File.OpenWrite("c:\\Work\\ganesh.parquet"))
{
using (var parquetWriter = new ParquetWriter(schema, fileStream))
{
// create a new row group in the file
using (ParquetRowGroupWriter groupWriter = parquetWriter.CreateRowGroup())
{
groupWriter.WriteColumn(idColumn);
groupWriter.WriteColumn(cityColumn);
groupWriter.WriteColumn(itemIDColumn);
groupWriter.WriteColumn(retailerIDColumn);
// groupWriter.WriteColumn(ratingDateTimeColumn);
groupWriter.WriteColumn(ratingValueColumn);
groupWriter.WriteColumn(itemUPCColumn);
groupWriter.WriteColumn(greceiptIDColumn);
groupWriter.WriteColumn(itemNameColumn);
groupWriter.WriteColumn(ratingLabelColumn);
groupWriter.WriteColumn(retailerTypeColumn);
}
}
}
