Implements failfast for transform and load errors for publications an…

…d grants.

closes #358

lib/rialto/etl/cli/grants.rb

@@ -100,10 +100,10 @@ def extract(row, profile_id, force)
             return extract_file if row[:sunetid].empty? # return from extract without calling SERA API
             results = perform_extract(row[:sunetid])
           rescue StandardError => exception
-            log_exception "Skipping #{extract_file} because an error occurred while extracting: " \
+            log_exception "Failing on #{profile_id} because an error occurred while extracting: " \
                           "#{exception.message} (#{exception.class})"
             FileUtils.rm(extract_file, force: true)
-            return extract_file
+            raise
           end
 
           return extract_file if results.empty? # this will avoid creating an empty file
@@ -129,10 +129,10 @@ def transform(source_file, profile_id, force)
               output_file_path: sparql_file
             ).transform
           rescue StandardError => exception
-            log_exception "Skipping #{sparql_file} because an error occurred while transforming: "\
+            log_exception "Failing on #{source_file} because an error occurred while transforming: "\
                           "#{exception.message} (#{exception.class})"
             FileUtils.rm(sparql_file, force: true)
-            return sparql_file
+            raise
           end
           sparql_file
         end

lib/rialto/etl/cli/publications.rb

@@ -100,8 +100,10 @@ def transform(source_file)
               output_file_path: sparql_file
             ).transform
           rescue StandardError => exception
-            say "Skipping #{sparql_file} because an error occurred while transforming: #{exception.message} (#{exception.class})"
+            log_exception "Failing on #{source_file} because an error occurred while transforming: " \
+              "#{exception.message} (#{exception.class})"
             FileUtils.rm(sparql_file, force: true)
+            raise
           end
           sparql_file
         end
@@ -112,7 +114,9 @@ def load_sparql(sparql_file)
           say "Loading sparql from #{sparql_file}"
           Rialto::Etl::Loaders::Sparql.new(input: sparql_file).load
         rescue StandardError => exception
-          say "An error occurred loading #{sparql_file} but continuing: #{exception.message} (#{exception.class})"
+          log_exception "Failing on #{sparql_file} because an error occurred while loading: " \
+            "#{exception.message} (#{exception.class})"
+          raise
         end
 
         def input_directory

spec/cli/grants_spec.rb

@@ -143,7 +143,7 @@
         File.open(sparql_file, 'w') { |file| file.write('test') }
         expect(File).to exist(sparql_file)
         # Using the SPARQL file has source file as a shortcut.
-        expect(loader.send(:transform, sparql_file, '1234', true)).to eq(sparql_file)
+        expect { loader.send(:transform, sparql_file, '1234', true) }.to raise_error(StandardError)
         expect(File).not_to exist(sparql_file)
         expect(Rialto::Etl::Transformer).to have_received(:new).once
         expect(Rialto::Etl::CLI::ErrorReporter).to have_received(:log_exception)
@@ -167,7 +167,7 @@
         ndj_file = File.join(dir, 'sera-1234.ndj')
         File.open(ndj_file, 'w') { |file| file.write('test') }
         expect(File).to exist(ndj_file)
-        expect(loader.send(:extract, row, '1234', true)).to eq(ndj_file)
+        expect { loader.send(:extract, row, '1234', true) }.to raise_error(StandardError)
         expect(Rialto::Etl::Extractors::Sera).to have_received(:new)
           .with(sunetid: 'vjarrett')
         expect(File).not_to exist(ndj_file)

spec/cli/publications_spec.rb

@@ -122,20 +122,38 @@
       end
     end
 
-    context 'when exception is raised during transform' do
+    context 'when exception raised during transform' do
       before do
         allow(Rialto::Etl::Transformer).to receive(:new).and_raise(StandardError)
         # Want source file to exist and sparql file not to exist. This mimics
         # the transformation removing the sparql file.
         allow(File).to receive(:exist?).and_return(true, false, false)
+        allow(Rialto::Etl::CLI::ErrorReporter).to receive(:log_exception)
       end
 
-      it 'does not attempt a load' do
-        loader.invoke_command(command)
+      it 'raises an error' do
+        expect { loader.invoke_command(command) }.to raise_error(StandardError)
         expect(Rialto::Etl::Extractors::WebOfScience).to have_received(:new).once
         expect(Rialto::Etl::Transformer).to have_received(:new).once
         expect(Rialto::Etl::Loaders::Sparql).not_to have_received(:new)
           .with(input: "#{dir}/WOS:000424386600014.sparql")
+        expect(Rialto::Etl::CLI::ErrorReporter).to have_received(:log_exception)
+      end
+    end
+
+    context 'when exception raised during load' do
+      before do
+        allow(Rialto::Etl::Loaders::Sparql).to receive(:new).and_raise(StandardError)
+        allow(Rialto::Etl::CLI::ErrorReporter).to receive(:log_exception)
+      end
+
+      it 'raises an error' do
+        expect { loader.invoke_command(command) }.to raise_error(StandardError)
+        expect(Rialto::Etl::Extractors::WebOfScience).to have_received(:new).once
+        expect(Rialto::Etl::Transformer).to have_received(:new).once
+        expect(Rialto::Etl::Loaders::Sparql).to have_received(:new)
+          .with(input: "#{dir}/WOS:000424386600014.sparql")
+        expect(Rialto::Etl::CLI::ErrorReporter).to have_received(:log_exception)
       end
     end
   end