Handle bom csv

Gemfile.lock

@@ -103,10 +103,10 @@ GEM
     aes_key_wrap (1.1.0)
     ast (2.4.2)
     aws-eventstream (1.3.0)
-    aws-partitions (1.1041.0)
+    aws-partitions (1.1043.0)
     aws-record (2.13.2)
       aws-sdk-dynamodb (~> 1, >= 1.85.0)
-    aws-sdk-core (3.216.0)
+    aws-sdk-core (3.217.0)
       aws-eventstream (~> 1, >= 1.3.0)
       aws-partitions (~> 1, >= 1.992.0)
       aws-sigv4 (~> 1.9)
@@ -171,7 +171,7 @@ GEM
     bindex (0.8.1)
     bootsnap (1.18.4)
       msgpack (~> 1.2)
-    brakeman (6.2.2)
+    brakeman (7.0.0)
       racc
     builder (3.3.0)
     bullet (8.0.0)
@@ -224,7 +224,7 @@ GEM
     docile (1.4.1)
     dotenv (3.1.7)
     drb (2.2.1)
-    dumb_delegator (1.0.0)
+    dumb_delegator (1.1.0)
     erubi (1.13.1)
     excon (1.2.3)
     factory_bot (6.5.0)
@@ -318,7 +318,7 @@ GEM
     kaminari-core (1.2.2)
     kramdown (2.5.1)
       rexml (>= 3.3.9)
-    language_server-protocol (3.17.0.3)
+    language_server-protocol (3.17.0.4)
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
@@ -420,7 +420,7 @@ GEM
       date
       stringio
     public_suffix (6.0.1)
-    puma (6.5.0)
+    puma (6.6.0)
       nio4r (~> 2.0)
     racc (1.8.1)
     rack (3.1.8)
@@ -519,7 +519,7 @@ GEM
     rspec-support (3.13.2)
     rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (1.70.0)
+    rubocop (1.71.0)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
@@ -529,9 +529,9 @@ GEM
       rubocop-ast (>= 1.36.2, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.37.0)
+    rubocop-ast (1.38.0)
       parser (>= 3.3.1.0)
-    rubocop-rails (2.29.0)
+    rubocop-rails (2.29.1)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.52.0, < 2.0)

app/controllers/admin/cx_collection_details_controller.rb

@@ -74,41 +74,50 @@ def export_csv
   # Handle a large-ish csv upload (5+ MB) to S3
   def upload_csv
     file = params[:file] # Assuming the file comes from a form field named 'file'
-
     file_extension = File.extname(file.original_filename).downcase
-
     @valid_file_extension = (file_extension == ".csv")
 
-    # check the file to ensure it is valid
-    csv_file = CSV.parse(file.read, headers: true)
-    begin
-    @valid_file_headers = csv_file.headers.sort == [
-      "external_id",
-      "question_1",
-      "positive_effectiveness",
-      "positive_ease",
-      "positive_efficiency",
-      "positive_transparency",
-      "positive_humanity",
-      "positive_employee",
-      "positive_other",
-      "negative_effectiveness",
-      "negative_ease",
-      "negative_efficiency",
-      "negative_transparency",
-      "negative_humanity",
-      "negative_employee",
-      "negative_other",
-      "question_4"
-    ].sort
-    rescue CSV::MalformedCSVError => e
-      flash[:alert] = "There was an error processing the CSV file: #{e.message}"
-      @valid_file_headers = false
-    rescue
-      @valid_file_headers = false
+    @valid_file_encoding = false
+    file_contents = file.read
+
+    @valid_file_encoding = file_contents.encoding.name == "UTF-8" && content.valid_encoding?
+    unless @valid_file_encoding
+      # Some CSVs may be encoded with a byte-order mark (BOM), so force encode again to remove
+      file_contents = file_contents.encode("UTF-8", invalid: :replace, undef: :replace, replace: "")
+      @valid_file_encoding = file_contents.encoding.name == "UTF-8" && file_contents.valid_encoding?
+    end
+
+    if @valid_file_encoding
+      csv_file = CSV.parse(file_contents, headers: true)
+      begin
+      @valid_file_headers = csv_file.headers.sort == [
+        "external_id",
+        "question_1",
+        "positive_effectiveness",
+        "positive_ease",
+        "positive_efficiency",
+        "positive_transparency",
+        "positive_humanity",
+        "positive_employee",
+        "positive_other",
+        "negative_effectiveness",
+        "negative_ease",
+        "negative_efficiency",
+        "negative_transparency",
+        "negative_humanity",
+        "negative_employee",
+        "negative_other",
+        "question_4"
+      ].sort
+      rescue CSV::MalformedCSVError => e
+        flash[:alert] = "There was an error processing the CSV file: #{e.message}"
+        @valid_file_headers = false
+      rescue
+        @valid_file_headers = false
+      end
     end
 
-    if @valid_file_extension && @valid_file_headers
+    if @valid_file_encoding && @valid_file_extension && @valid_file_headers
       key = "cx_data_collections/cx-upload-#{timestamp_string}-#{file.original_filename}"
       obj = s3_bucket.object(key)
       # Upload the file
@@ -124,13 +133,14 @@ def upload_csv
       Event.log_event(Event.names[:cx_collection_detail_upload_created], @cxdu.class.to_s, @cxdu.id, "CX Collection Detail Upload #{@cxdu.id} created at #{DateTime.now}", current_user.id)
 
       flash[:notice] = "A .csv file with #{csv_file.size} rows was uploaded successfully. Please see your uploaded file in the table below, then return to the CX Data Collection."
+    elsif !@valid_file_encoding
+      flash[:alert] = "The uploaded file must be encoded as UTF-8"
     elsif !@valid_file_extension
       flash[:notice] = "File has a file extension of #{file_extension}, but it should be .csv."
     elsif !@valid_file_headers
       flash[:alert] = "CSV headers do not match. Headers received were: #{csv_file.headers.to_s}"
     end
 
-    # render :upload
     redirect_to upload_admin_cx_collection_detail_path(@cx_collection_detail)
   end
 

spec/fixtures/sample_cx_responses_upload_with_bom.csv

@@ -0,0 +1,5 @@
+external_id,question_1,positive_effectiveness,positive_ease,positive_efficiency,positive_transparency,positive_humanity,positive_employee,positive_other,negative_effectiveness,negative_ease,negative_efficiency,negative_transparency,negative_humanity,negative_employee,negative_other,question_4
+100001,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,"alright"
+100002,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,""
+100003,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,Excellent listening
+100004,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,"Very intuitive experience, easy!"