add script to run all crawls and generate csv

README.md

@@ -37,6 +37,10 @@ Run a spider:
 
     scrapy crawl us.pycon.org
 
+Run all spiders and generate a data.csv file:
+
+    run.sh
+
 Scrapy Cloud Test Project
 -------------------------
 

run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+for spider in $(scrapy list)
+do
+    # put all the data in separate files to make it easier to trace
+    # data back to the spider
+    scrapy crawl $spider -o data/$spider.csv -t csv
+done
+
+# dedupe and merge
+
+# should generate a single header (if not, we have inconsistent data)
+head -n 1 -q data/*.csv | sort -u > alldata.csv
+tail -q -n +2 data/*.csv | sort -u >> data.csv