diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ea1307e2e..b75517655 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,14 +23,14 @@ jobs: cd docker && bash ./dockbuild.sh - name: Build & Run Python Tests via Docker - run: cd docker && ./testall python + run: cd docker && ./testall -D python - name: Build & Run Java Tests via Docker - run: cd docker && ./makedist java + run: cd docker && ./makedist -D java - name: Build Angular Code - run: cd docker && ./makedist angular + run: cd docker && ./makedist -D angular - name: Run Angular Tests - run: cd docker && ./testall angular + run: cd docker && ./testall -D angular diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e20042e6e..b40d59475 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,14 +23,14 @@ jobs: cd docker && bash ./dockbuild.sh - name: Build & Run Python Tests via Docker - run: cd docker && ./testall python + run: cd docker && ./testall -D python - name: Build & Run Java Tests via Docker - run: cd docker && ./makedist java + run: cd docker && ./makedist -D java - name: Build Angular Code - run: cd docker && ./makedist angular + run: cd docker && ./makedist -D angular - name: Run Angular Tests - run: cd docker && ./testall angular + run: cd docker && ./testall -D angular diff --git a/.github/workflows/python-source.yml b/.github/workflows/python-source.yml index 58a6e9925..becaf0752 100644 --- a/.github/workflows/python-source.yml +++ b/.github/workflows/python-source.yml @@ -27,5 +27,5 @@ jobs: cd docker && bash ./dockbuild.sh python - name: Run Unit Tests via Docker - run: cd docker && ./testall python + run: cd docker && ./testall -D python diff --git a/.github/workflows/testall.yml b/.github/workflows/testall.yml index 2e34ec0ca..b85649779 100644 --- a/.github/workflows/testall.yml +++ b/.github/workflows/testall.yml @@ -27,13 +27,13 @@ jobs: cd docker && bash ./dockbuild.sh - name: Build & Run Python Tests via Docker - run: cd docker && ./testall python + run: cd docker && ./testall -D python - name: Test Java via Docker - run: cd docker && ./testall java + run: cd docker && ./testall -D java - name: Build Angular Code - run: cd docker && ./makedist angular + run: cd docker && ./makedist -D angular - name: Run Angular Tests - run: cd docker && ./testall angular + run: cd docker && ./testall -D angular diff --git a/angular/package.json b/angular/package.json index 817575412..e411a39ec 100644 --- a/angular/package.json +++ b/angular/package.json @@ -87,7 +87,7 @@ "jasmine-core": "~3.8.0", "jasmine-spec-reporter": "~5.0.0", "karma": "^6.3.16", - "karma-chrome-launcher": "~3.1.0", + "karma-chrome-launcher": "~3.2.0", "karma-coverage-istanbul-reporter": "~2.0.0", "karma-jasmine": "~4.0.0", "karma-jasmine-html-reporter": "^1.6.0", diff --git a/docker/angtest/Dockerfile b/docker/angtest/Dockerfile index 7532b9324..f471ed6ed 100644 --- a/docker/angtest/Dockerfile +++ b/docker/angtest/Dockerfile @@ -8,7 +8,8 @@ RUN set -ex; \ wget https://dl-ssl.google.com/linux/linux_signing_key.pub; \ apt-key add linux_signing_key.pub; \ apt-get update; \ - apt-get install -y --no-install-recommends google-chrome-stable less; \ - ln -s /usr/bin/google-chrome-stable /usr/bin/chrome + apt-get install -y --no-install-recommends chromium less; \ + ln -s /usr/bin/chromium /usr/bin/chrome +ENV CHROME_BIN /usr/bin/chromium ENTRYPOINT ["docker/angtest/entrypoint.sh"] diff --git a/docker/cacerts/README.md b/docker/cacerts/README.md new file mode 100644 index 000000000..ceae86eb2 --- /dev/null +++ b/docker/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. diff --git a/docker/dockbuild.sh b/docker/dockbuild.sh index e5f865e5f..7f86648bc 100755 --- a/docker/dockbuild.sh +++ b/docker/dockbuild.sh @@ -44,6 +44,9 @@ if { echo " $BUILD_IMAGES " | grep -qs " pdrtest "; }; then docker build $BUILD_OPTS -t $PACKAGE_NAME/pdrtest pdrtest 2>&1 | logit fi if { echo " $BUILD_IMAGES " | grep -qs " pdrangular "; }; then + # install CA certs into containers that can use them + cp_ca_certs_to pdrangular + echo '+' docker build $BUILD_OPTS -t $PACKAGE_NAME/pdrangular pdrangular docker build $BUILD_OPTS -t $PACKAGE_NAME/pdrangular pdrangular 2>&1 fi diff --git a/docker/pdrangular/Dockerfile b/docker/pdrangular/Dockerfile index 5b5f2cd90..4f4358188 100644 --- a/docker/pdrangular/Dockerfile +++ b/docker/pdrangular/Dockerfile @@ -19,15 +19,23 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends zip wget ca-certificates git xz-utils \ gnupg python +COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates +ENV NODE_EXTRA_CA_CERTS /etc/ssl/certs/ca-certificates.crt + ARG node_ver=v14.19.0 -COPY node-sha256.txt /tmp -RUN cd /tmp && wget https://nodejs.org/dist/$node_ver/node-$node_ver-linux-x64.tar.xz \ - && sha256sum -c node-sha256.txt -RUN cd /usr/local && tar xJf /tmp/node-$node_ver-linux-x64.tar.xz \ - && ln -s node-$node_ver-linux-x64 node \ - && cd bin \ - && ln -s ../node/bin/node node \ - && ln -s ../node/bin/npm npm \ +COPY node-*-sha256.txt /tmp/ +RUN arch=`dpkg --print-architecture` && \ + { [ "$arch" != "amd64" ] || arch="x64"; } && echo $arch > /etc/plat-arch +RUN cd /tmp && arch=`cat /etc/plat-arch` \ + && wget https://nodejs.org/dist/$node_ver/node-$node_ver-linux-${arch}.tar.xz \ + && sha256sum -c node-${arch}-sha256.txt +RUN cd /usr/local && arch=`cat /etc/plat-arch` \ + && tar xJf /tmp/node-$node_ver-linux-${arch}.tar.xz \ + && ln -s node-$node_ver-linux-${arch} node \ + && cd bin \ + && ln -s ../node/bin/node node \ + && ln -s ../node/bin/npm npm \ && ln -s ../node/bin/npx npx ENV GOSU_VERSION 1.14 diff --git a/docker/pdrangular/cacerts/README.md b/docker/pdrangular/cacerts/README.md new file mode 100644 index 000000000..ceae86eb2 --- /dev/null +++ b/docker/pdrangular/cacerts/README.md @@ -0,0 +1,13 @@ +This directory contains non-standard CA certificates needed to build the docker +images. + +Failures building the Docker containers defined in ../ due to SSL certificate +verification errors may be a consequence of your local network's firewall. In +particular, the firewall may be substituting external site certificates with +its own signed by a non-standard CA certficate (chain). If so, you can place +the necessary certificates into this directory; they will be passed into the +containers, allowing them to safely connect to those external sites. + +Be sure the certificates are in PEM format and include a .crt file extension. + +Do not remove this README file; doing so may cause a Docker build faiure. diff --git a/docker/pdrangular/node-arm64-sha256.txt b/docker/pdrangular/node-arm64-sha256.txt new file mode 100644 index 000000000..39a92aa93 --- /dev/null +++ b/docker/pdrangular/node-arm64-sha256.txt @@ -0,0 +1 @@ +73224b331d2bb0f05fd2ff62d1c736460f12f8e8757e8afbbcdf11a7f1464da5 node-v14.19.0-linux-arm64.tar.xz diff --git a/docker/pdrangular/node-sha256.txt b/docker/pdrangular/node-x64-sha256.txt similarity index 100% rename from docker/pdrangular/node-sha256.txt rename to docker/pdrangular/node-x64-sha256.txt diff --git a/docker/run.sh b/docker/run.sh index 2921b5be8..925e12e04 100755 --- a/docker/run.sh +++ b/docker/run.sh @@ -76,6 +76,9 @@ while [ "$1" != "" ]; do -d|--docker-build) dodockbuild=1 ;; + -D|--no-docker-build) + dodockbuild=0 + ;; --dist-dir) shift distdir="$1" @@ -172,7 +175,7 @@ if [ -z "$dodockbuild" ]; then fi fi -[ -z "$dodockbuild" ] || { +[ "$dodockbuild" != "1" ] || { echo '#' Building missing docker containers... $execdir/dockbuild.sh } diff --git a/oar-build/_dockbuild.sh b/oar-build/_dockbuild.sh index f4ba89422..88e41d62d 100644 --- a/oar-build/_dockbuild.sh +++ b/oar-build/_dockbuild.sh @@ -60,6 +60,21 @@ function setup_build { BUILD_OPTS=`collect_build_opts` } +function cp_ca_certs_to { + # assuming we are in the docker dir + [ \! -d cacerts ] || { + crts=`compgen -G 'cacerts/*.crt' || true` + [ -z "$crts" ] || { + echo "${prog}: installing CA certs from docker/cacerts" + for cont in $@; do + mkdir -p $cont/cacerts + echo '+' cp $crts cacerts/README.md $cont/cacerts + cp $crts cacerts/README.md $cont/cacerts + done + } + } +} + function help { helpfile=$OAR_BUILD_DIR/dockbuild_help.txt [ -f "$OAR_DOCKER_DIR/dockbuild_help.txt" ] && \ diff --git a/oar-metadata b/oar-metadata index 7f4196d23..e8237b4b7 160000 --- a/oar-metadata +++ b/oar-metadata @@ -1 +1 @@ -Subproject commit 7f4196d232af152f7c62150163607747ac9ca83a +Subproject commit e8237b4b71c9a4b757f4d7fb5f59d7fec78355e7 diff --git a/python/tests/nistoar/pdr/preserv/data/simplesip/_nerdm.json b/python/tests/nistoar/pdr/preserv/data/simplesip/_nerdm.json index ab6f28764..1173d39a2 100644 --- a/python/tests/nistoar/pdr/preserv/data/simplesip/_nerdm.json +++ b/python/tests/nistoar/pdr/preserv/data/simplesip/_nerdm.json @@ -1,8 +1,8 @@ { "@context": "https://data.nist.gov/od/dm/nerdm-pub-context.jsonld", - "_schema": "https://data.nist.gov/od/dm/nerdm-schema/v0.6#", + "_schema": "https://data.nist.gov/od/dm/nerdm-schema/v0.7#", "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/PublishedDataResource" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/PublishedDataResource" ], "@type": [ "nrdp:PublishedDataResource" @@ -58,7 +58,7 @@ "refType": "IsReferencedBy", "location": "https://doi.org/10.1364/OE.24.014100", "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/bib/v0.6#/definitions/DCiteDocumentReference" + "https://data.nist.gov/od/dm/nerdm-schema/bib/v0.7#/definitions/DCiteDocumentReference" ] } ], @@ -76,7 +76,7 @@ "dcat:Distribution" ], "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile" ] }, { @@ -89,7 +89,7 @@ "dcat:Distribution" ], "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile" ] }, { @@ -113,7 +113,7 @@ "dcat:Distribution" ], "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile" ] }, { @@ -126,7 +126,7 @@ "dcat:Distribution" ], "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile" ] }, { @@ -139,7 +139,7 @@ "dcat:Distribution" ], "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile" ] }, { @@ -152,7 +152,7 @@ "dcat:Distribution" ], "_extensionSchemas": [ - "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile" + "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile" ] } ], diff --git a/scripts/install_ca_certs.sh b/scripts/install_ca_certs.sh new file mode 100755 index 000000000..c759a6d17 --- /dev/null +++ b/scripts/install_ca_certs.sh @@ -0,0 +1,37 @@ +#! /bin/bash +# +# install_ca_certs.sh -- copy the specified CA certificates into this source so that they can be used +# to build the software via docker. +# +# usage: install_ca_certs.sh CA_CERT_FILE... +# +# where CA_CERT_FILE is a file path to a CA certificate to install +# +# This script helps address the problem with docker-based builds when run within a firewall that +# replaces external site certificates with ones signed by a non-standard CA, causing the retrieval +# of software dependencies to fail. This script is used by oar-docker's localbuild script to receive +# extra CA certificates that addresses such failures. Because localdeploy makes no assumptions about +# how this source code repository builds using docker, this script encapsulates that knowledge on +# behalf of localbuild. +# +# Note: if this repository does not require/support use of non-standard CA certificates, remove (or +# rename) this script. +# +set -e +prog=`basename $0` +execdir=`dirname $0` +[ "$execdir" = "" -o "$execdir" = "." ] && execdir=$PWD +basedir=`dirname $execdir` + +cacertdir="$basedir/docker/cacerts" +[ -d "$cacertdir" ] || exit 0 # I guess we don't need the certs + +crts=`echo $@ | sed -e 's/^ *//' -e 's/ *$//'` +[ -n "$crts" ] || { + print "${prog}: Missing cert file argument" + false +} + +echo '+' cp $crts $cacertdir +cp $crts $cacertdir +