Skip to content

Commit

Permalink
Support decimal type properly
Browse files Browse the repository at this point in the history
Redshift handles Decimal as string currently when loading data from Avro format since Redshift used older version of Avro
The Avro supported Decimal type natively from 1.7.7 - https://issues.apache.org/jira/browse/AVRO-1402
  • Loading branch information
sungjuly committed Jan 31, 2019
1 parent 184b442 commit 858abe4
Showing 1 changed file with 3 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ private[redshift] class RedshiftWriter(
// However, each task gets its own deserialized copy, making this safe.
val conversionFunctions: Array[Any => Any] = data.schema.fields.map { field =>
field.dataType match {
case _: DecimalType => (v: Any) => if (v == null) null else v.toString
case DateType =>
val dateFormat = Conversions.createRedshiftDateFormat()
(v: Any) => {
Expand Down Expand Up @@ -271,6 +272,8 @@ private[redshift] class RedshiftWriter(
// strings. This is necessary for Redshift to be able to load these columns (see #39).
val convertedSchema: StructType = StructType(
schemaWithLowercaseColumnNames.map {
case StructField(name, _: DecimalType, nullable, meta) =>
StructField(name, StringType, nullable, meta)
case StructField(name, DateType, nullable, meta) =>
StructField(name, StringType, nullable, meta)
case StructField(name, TimestampType, nullable, meta) =>
Expand Down

0 comments on commit 858abe4

Please sign in to comment.