CI corrections with ckan 2.9+ prework (#18)

* ci build
* correct psycopg2 for later generations of db, ignore masters for now
* change os for 2.7 build
* handle jetty8 and jetty9
* add xlsx tests, tweak for bionic builds libmagic works differently for xls files
* skip xls tests due to libmagic1 changes in os

libmagic1 that works out xls is not as forgiving as it once was, the two tests
were checking to see if messytables was used or an error thrown on partial load
since xls format stopped in 2004, and xlsx is the defacto format for excel
this might be ok to ignore now as it still works on Trusty LTS

Bionic
libmagic1:
  Installed: 1:5.32-2ubuntu0.4
  Candidate: 1:5.32-2ubuntu0.4
  Version table:
 *** 1:5.32-2ubuntu0.4 500
        500 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages
        100 /var/lib/dpkg/status

Trusty
libmagic1:
  Installed: 1:5.14-2ubuntu3.3
  Candidate: 1:5.14-2ubuntu3.3
  Version table:
 *** 1:5.14-2ubuntu3.3 500
        500 http://security.ubuntu.com/ubuntu trusty-security/main amd64 Packages
        100 /var/lib/dpkg/status


* ckan 2.9+ compatibilities

  paster is replaced with ckan command from 2.9+
  unicode, basestring is now just str in python3
  deal with version and python differences in the cicd build

* python3 flake
  ur not supported in ptyon3, use br"".decode('raw_unicode_escape')

.gitignore

@@ -44,3 +44,8 @@ docs/_build/
 # editors
 .vscode
 
+ckan
+.rdp
+.noseids
+subdir/
+dump.rdb

.travis.yml

@@ -1,22 +1,65 @@
+os: linux
 language: python
-
-# the new trusty images of Travis cause build errors with psycopg2, see https://github.com/travis-ci/travis-ci/issues/8897
-dist: trusty
-group: deprecated-2017Q4
-
+dist: bionic
 python:
     - "2.7"
 env:
-    - CKANVERSION=master
-    - CKANVERSION=2.8
-    - CKANVERSION=2.7
-install:
-    - bash bin/travis-build.bash
+    - CKAN_GIT_REPO=ckan CKAN_BRANCH=master
+    - CKAN_GIT_REPO=ckan CKANVERSION=2.8
+    - CKAN_GIT_REPO=qld-gov-au CKAN_BRANCH=qgov-master
 services:
-    - postgresql
     - redis
-addons:
-    postgresql: "9.3"
+    - postgresql
+before_install:
+    - apt-cache policy libmagic1
+install:
+    - pip install -U pip wheel
+    - bash bin/travis-build.bash
+    - pip install coveralls
 script: sh bin/travis-run.sh
-after_success: coveralls
-sudo: required
+after_success:
+    - coveralls
+
+stages:
+  - Flake8
+  - test
+
+jobs:
+  include:
+    - stage: Flake8
+      env: FLAKE8=True
+      install:
+        - pip install -U pip wheel
+        - sh bin/travis-flake.sh
+      script:
+        -  sh bin/travis-flake-run.sh
+    - stage: test #master build on python 3 alpha
+      env: CKAN_GIT_REPO=ckan CKAN_BRANCH=master
+      python: "3.6"
+    - stage: test #2.7 build on trusty
+      # ensure https://travis-ci.org/github/$yourRepo/ckanext-xloader/settings "Enable build config validation" is off if builds are not getting to testing
+      # the new trusty images of Travis cause build errors with psycopg2 in 2.7 and below, see https://github.com/travis-ci/travis-ci/issues/8897
+      # due to psycopg2 2.4 does not support newer db's, https://github.com/psycopg/psycopg2/issues/594 and no backports being given to 2.7 or older
+      # releases, we need ot hard lock for them. (needs to be on libpq-dev=9.3.* )
+      dist: trusty
+      group: deprecated-2017Q4 #must be upper case but travis "build config validation" if enabled will lowercase it and break the build
+      env: CKAN_GIT_REPO=ckan CKANVERSION=2.7
+      addons:
+        postgresql: "9.6"
+    - stage: test #2.8 build on trusty
+      # ensure https://travis-ci.org/github/$yourRepo/ckanext-xloader/settings "Enable build config validation" is off if builds are not getting to testing
+      # the new trusty images of Travis cause build errors with psycopg2 in 2.7 and below, see https://github.com/travis-ci/travis-ci/issues/8897
+      # due to psycopg2 2.4 does not support newer db's, https://github.com/psycopg/psycopg2/issues/594 and no backports being given to 2.7 or older
+      # releases, we need ot hard lock for them. (needs to be on libpq-dev=9.3.* )
+      dist: trusty
+      group: deprecated-2017Q4 #must be upper case but travis "build config validation" if enabled will lowercase it and break the build
+      env: CKAN_GIT_REPO=ckan CKANVERSION=2.8
+      addons:
+        postgresql: "9.6"
+  allow_failures:
+    - env: CKAN_GIT_REPO=ckan CKAN_BRANCH=master
+      python: "3.6"
+    - env: CKAN_GIT_REPO=ckan CKAN_BRANCH=master
+      python: "2.7" #master build on python 2 alpha (is it backwards compatible)
+
+cache: pip

bin/localPdev.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -ex
+# please start up docker via
+# docker run -it -v "`pwd`":/build ubuntu:bionic /bin/bash
+# cd /build ;bash -x  /build/bin/localPdev.sh
+
+
+
+apt-get update
+apt-get install -y sudo systemd postgresql-10 git python python-pip redis-server
+/etc/init.d/postgresql start
+redis-server &
+
+pip install -U pip wheel
+
+export CKAN_GIT_REPO=ckan/ckan
+export CKANVERSION=2.8
+
+bash -x bin/travis-build.bash
+
+bash -x bin/travis-run.sh

bin/travis-build.bash

@@ -5,76 +5,138 @@ echo "This is travis-build.bash..."
 
 echo "Installing the packages that CKAN requires..."
 sudo apt-get update -qq
-sudo apt-get install solr-jetty
+sudo apt-get install -y solr-jetty libcommons-fileupload-java
 
-echo "Installing CKAN and its Python dependencies..."
-git clone https://github.com/ckan/ckan
-cd ckan
-if [ $CKANVERSION == 'master' ]
-then
-    echo "CKAN version: master"
+# !/bin/bash
+ver=$(python -c"import sys; print(sys.version_info.major)")
+if [ $ver -eq 2 ]; then
+    echo "python version 2"
+elif [ $ver -eq 3 ]; then
+    echo "python version 3"
 else
-    CKAN_TAG=$(git tag | grep ^ckan-$CKANVERSION | sort --version-sort | tail -n 1)
-    git checkout $CKAN_TAG
-    echo "CKAN version: ${CKAN_TAG#ckan-}"
+    echo "Unknown python version: $ver"
+fi
+
+cliCommand=paster
+
+echo "Installing CKAN and its Python dependencies..."
+if [ ! -d ckan ]; then
+
+  echo "Creating the PostgreSQL user and database..."
+  sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+  sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+  sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
+  sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER datastore_default;'
+
+  echo "Create full text function..."
+  cp full_text_function.sql /tmp
+  pushd /tmp
+  sudo -u postgres psql datastore_test -f full_text_function.sql
+  popd
+
+  if [ "${CKAN_BRANCH}dd" == 'dd' ]; then
+    #remote lookup tags, and get latest by version-sort
+    CKAN_TAG=$(git ls-remote --tags https://github.com/$CKAN_GIT_REPO/ckan | grep refs/tags/ckan-$CKANVERSION | awk '{print $2}'| sort --version-sort | tail -n 1 | sed  's|refs/tags/||' )
+    echo "CKAN tag version $CKANVERSION is: ${CKAN_TAG#ckan-}"
+    git clone --depth=50 --branch=$CKAN_TAG https://github.com/$CKAN_GIT_REPO/ckan ckan
+    if [ $CKANVERSION \< '2.9' ]
+    then
+       cliCommand=paster
+    else
+        cliCommand=ckan
+    fi
+  else
+    echo "CKAN version: $CKAN_BRANCH"
+    git clone --depth=50 --branch=$CKAN_BRANCH https://github.com/$CKAN_GIT_REPO/ckan ckan
+    #Master is on 2.9+ so needs ckan for cliCommand
+    if [ $CKAN_BRANCH == 'master' ]
+    then
+       cliCommand=ckan
+    fi
+  fi
 fi
 
+pushd ckan
+
+if [ $ver -eq 3 ]; then
 # install the recommended version of setuptools
 if [ -f requirement-setuptools.txt ]
 then
     echo "Updating setuptools..."
     pip install -r requirement-setuptools.txt
 fi
+fi
 
 if [ $CKANVERSION == '2.7' ]
 then
     echo "Installing setuptools"
     pip install setuptools==39.0.1
 fi
 
-python setup.py develop
-if [ -f requirements-py2.txt ]
-then
+if [ -f requirements-py2.txt ] && [ $ver -eq 2 ]; then
     pip install -r requirements-py2.txt
 else
     pip install -r requirements.txt
 fi
 pip install -r dev-requirements.txt
-cd -
+python setup.py develop
 
-echo "Creating the PostgreSQL user and database..."
-sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
-sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER datastore_default;'
+echo "Initialising the database..."
+if [ "$cliCommand" == "paster" ]; then
+  paster db init -c test-core.ini
+  paster datastore set-permissions -c test-core.ini | sudo -u postgres psql
+else
+  #ckan comand has config first then options.
+  ckan -c test-core.ini db init
+  ckan -c test-core.ini datastore set-permissions  | sudo -u postgres psql
+fi
+popd
 
-echo "Setting up Solr..."
-# Solr is multicore for tests on ckan master, but it's easier to run tests on
-# Travis single-core. See https://github.com/ckan/ckan/issues/2972
-sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini
-printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+
+echo "SOLR config..."
 sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
-sudo service jetty restart
 
-echo "Create full text function..."
-cp full_text_function.sql /tmp
-cd /tmp
-sudo -u postgres psql datastore_test -f full_text_function.sql
-cd -
+# solr is multicore for tests on ckan master now, but it's easier to run tests
+# on Travis single-core still.
+# see https://github.com/ckan/ckan/issues/2972
+sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini
 
-echo "Initialising the database..."
-cd ckan
-paster db init -c test-core.ini
-paster datastore set-permissions -c test-core.ini | sudo -u postgres psql
-cd -
 
 echo "Installing ckanext-xloader and its requirements..."
-python setup.py develop
-pip install -r requirements.txt
+if [ -f requirements-py2.txt ] && [ $ver -eq 2 ]; then
+  pip install -r requirements-py2.txt
+elif [ -f requirements.txt ]; then
+  pip install -r requirements.txt
+fi
 pip install -r dev-requirements.txt
+python setup.py develop
 
 echo "Moving test.ini into a subdir..."
-mkdir subdir
-mv test.ini subdir
+mkdir -p subdir
+cp test.ini subdir
+
+echo "start solr on $TRAVIS_DIST"
+
+#ensure we handle jetty8 and jetty9
+if [[ "$TRAVIS_DIST" == "trusty" ]]; then
+  printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+  sudo service jetty restart
+
+else
+  # expect we are in current os, i.e. bionic
+  #[ "$TRAVIS_DIST" == "bionic" ]; then
+  # Fix solr-jetty starting issues https://stackoverflow.com/a/56007895
+  # https://github.com/Zharktas/ckanext-report/blob/py3/bin/travis-run.bash
+  sudo mkdir -p /etc/systemd/system/jetty9.service.d
+  printf "[Service]\nReadWritePaths=/var/lib/solr" | sudo tee /etc/systemd/system/jetty9.service.d/solr.conf
+  sed '16,21d' /etc/solr/solr-jetty.xml | sudo tee /etc/solr/solr-jetty.xml
+  sudo systemctl daemon-reload || echo "all good"
+  printf "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_ARGS=\"jetty.http.port=8983\"\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty9
+  sudo service jetty9 restart
+fi
+
+# Wait for jetty to start
+timeout 20 bash -c 'while [[ "$(curl -s -o /dev/null -I -w %{http_code} http://localhost:8983)" != "200" ]]; do sleep 2;done'
+
 
 echo "travis-build.bash is done."

bin/travis-flake-run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh -e
+flake8 --version
+# stop the build if there are Python syntax errors or undefined names
+flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,ckanext-xloader
+
+# exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
+flake8 . --count --max-complexity=27 --max-line-length=127 --statistics --exclude ckan,ckanext-xloader

bin/travis-flake.sh

@@ -0,0 +1,3 @@
+
+pip install -U pip wheel
+pip install -r flake-requirements.txt

bin/travis-run.sh

@@ -1,10 +1,30 @@
-#!/bin/sh -e
+#!/bin/bash
 set -ex
 
 flake8 --version
 # stop the build if there are Python syntax errors or undefined names
 flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,ckanext-xloader
 
+
+ver=$(python -c"import sys; print(sys.version_info.major)")
+testFramework=nose
+if [ "${CKAN_BRANCH}dd" == 'dd' ]; then
+  if [ "$CKANVERSION" == '2.9' ]
+    then
+       testFramework=pytest
+    else
+        testFramework=nose
+    fi
+elif [ "$CKAN_BRANCH" == 'master' ]; then
+       testFramework=pytest
+fi
+
+# shellcheck disable=SC1072
+if [[ $ver -eq 3  ||  "$testFramework" == "pytest" ]]; then
+    echo "python version 3 or 2.9+ ckan running pytest"
+    pytest --ckan-ini=subdir/test.ini --cov=ckanext.xloader ckanext/xloader/tests
+else
+    echo "python version 2 running nosetests"
 nosetests --ckan \
           --nologcapture \
           --with-pylons=subdir/test.ini \
@@ -13,6 +33,7 @@ nosetests --ckan \
           --cover-inclusive \
           --cover-erase \
           --cover-tests
+fi
 
 # strict linting
 flake8 . --count --max-complexity=27 --max-line-length=127 --statistics --exclude ckan,ckanext-xloader

ckanext/xloader/action.py

@@ -6,7 +6,7 @@
 
 from dateutil.parser import parse as parse_date
 
-import ckan.lib.helpers as h
+# import ckan.lib.helpers as h
 import ckan.lib.navl.dictization_functions
 import ckan.logic as logic
 import ckan.plugins as p
@@ -336,7 +336,7 @@ def xloader_status(context, data_dict):
         job_detail = db.get_job(job_id)
 
         # Attach time zone data to logs if needed
-        #job_detail['logs']  TypeError: 'NoneType' object has no attribute '__getitem__'
+        # job_detail['logs']  TypeError: 'NoneType' object has no attribute '__getitem__'
         # for log in job_detail['logs']:
         #     if 'timestamp' in log:
         #         date = log['timestamp']