Merge branch 'fix/docker-cacerts-m1' into feature/rpa-datacart

usnistgov · Dec 5, 2023 · 6259263 · 6259263
2 parents 0aa44d2 + 0304b3d
commit 6259263
Show file tree

Hide file tree

Showing 17 changed files with 129 additions and 35 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -23,14 +23,14 @@ jobs:
           cd docker && bash ./dockbuild.sh
 
       - name: Build & Run Python Tests via Docker
-        run: cd docker && ./testall python
+        run: cd docker && ./testall -D python
 
       - name: Build & Run Java Tests via Docker
-        run: cd docker && ./makedist java
+        run: cd docker && ./makedist -D java
 
       - name: Build Angular Code
-        run: cd docker && ./makedist angular
+        run: cd docker && ./makedist -D angular
 
       - name: Run Angular Tests
-        run: cd docker && ./testall angular
+        run: cd docker && ./testall -D angular
 
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -23,14 +23,14 @@ jobs:
           cd docker && bash ./dockbuild.sh
 
       - name: Build & Run Python Tests via Docker
-        run: cd docker && ./testall python
+        run: cd docker && ./testall -D python
 
       - name: Build & Run Java Tests via Docker
-        run: cd docker && ./makedist java
+        run: cd docker && ./makedist -D java
 
       - name: Build Angular Code
-        run: cd docker && ./makedist angular
+        run: cd docker && ./makedist -D angular
 
       - name: Run Angular Tests
-        run: cd docker && ./testall angular
+        run: cd docker && ./testall -D angular
 
diff --git a/.github/workflows/python-source.yml b/.github/workflows/python-source.yml
@@ -27,5 +27,5 @@ jobs:
           cd docker && bash ./dockbuild.sh python
 
       - name: Run Unit Tests via Docker
-        run: cd docker && ./testall python
+        run: cd docker && ./testall -D python
 
diff --git a/.github/workflows/testall.yml b/.github/workflows/testall.yml
@@ -27,13 +27,13 @@ jobs:
           cd docker && bash ./dockbuild.sh
 
       - name: Build & Run Python Tests via Docker
-        run: cd docker && ./testall python
+        run: cd docker && ./testall -D python
 
       - name: Test Java via Docker
-        run: cd docker && ./testall java
+        run: cd docker && ./testall -D java
 
       - name: Build Angular Code
-        run: cd docker && ./makedist angular
+        run: cd docker && ./makedist -D angular
 
       - name: Run Angular Tests
-        run: cd docker && ./testall angular
+        run: cd docker && ./testall -D angular
diff --git a/angular/package.json b/angular/package.json
@@ -87,7 +87,7 @@
     "jasmine-core": "~3.8.0",
     "jasmine-spec-reporter": "~5.0.0",
     "karma": "^6.3.16",
-    "karma-chrome-launcher": "~3.1.0",
+    "karma-chrome-launcher": "~3.2.0",
     "karma-coverage-istanbul-reporter": "~2.0.0",
     "karma-jasmine": "~4.0.0",
     "karma-jasmine-html-reporter": "^1.6.0",

diff --git a/docker/angtest/Dockerfile b/docker/angtest/Dockerfile
@@ -8,7 +8,8 @@ RUN set -ex; \
     wget https://dl-ssl.google.com/linux/linux_signing_key.pub; \
     apt-key add linux_signing_key.pub; \
     apt-get update; \
-    apt-get install -y --no-install-recommends google-chrome-stable less; \
-    ln -s /usr/bin/google-chrome-stable /usr/bin/chrome
+    apt-get install -y --no-install-recommends chromium less; \
+    ln -s /usr/bin/chromium /usr/bin/chrome
+ENV CHROME_BIN /usr/bin/chromium
 
 ENTRYPOINT ["docker/angtest/entrypoint.sh"]
diff --git a/docker/cacerts/README.md b/docker/cacerts/README.md
@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.
diff --git a/docker/dockbuild.sh b/docker/dockbuild.sh
@@ -44,6 +44,9 @@ if { echo " $BUILD_IMAGES " | grep -qs " pdrtest "; }; then
     docker build $BUILD_OPTS -t $PACKAGE_NAME/pdrtest pdrtest 2>&1 | logit
 fi
 if { echo " $BUILD_IMAGES " | grep -qs " pdrangular "; }; then
+    # install CA certs into containers that can use them
+    cp_ca_certs_to pdrangular
+
     echo '+' docker build $BUILD_OPTS -t $PACKAGE_NAME/pdrangular pdrangular
     docker build $BUILD_OPTS -t $PACKAGE_NAME/pdrangular pdrangular 2>&1
 fi

diff --git a/docker/pdrangular/Dockerfile b/docker/pdrangular/Dockerfile
@@ -19,15 +19,23 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends zip wget ca-certificates git xz-utils \
                                                gnupg python
 
+COPY cacerts/README.md cacerts/*.crt /usr/local/share/ca-certificates/
+RUN update-ca-certificates
+ENV NODE_EXTRA_CA_CERTS /etc/ssl/certs/ca-certificates.crt
+
 ARG node_ver=v14.19.0
-COPY node-sha256.txt /tmp
-RUN cd /tmp && wget https://nodejs.org/dist/$node_ver/node-$node_ver-linux-x64.tar.xz  \
-            && sha256sum -c node-sha256.txt
-RUN cd /usr/local && tar xJf /tmp/node-$node_ver-linux-x64.tar.xz \
-                  && ln -s node-$node_ver-linux-x64 node          \
-                  && cd bin                                     \
-                  && ln -s ../node/bin/node node                \
-                  && ln -s ../node/bin/npm  npm                 \
+COPY node-*-sha256.txt /tmp/
+RUN arch=`dpkg --print-architecture` && \
+    { [ "$arch" != "amd64" ] || arch="x64"; } && echo $arch > /etc/plat-arch
+RUN cd /tmp && arch=`cat /etc/plat-arch`  \
+            && wget https://nodejs.org/dist/$node_ver/node-$node_ver-linux-${arch}.tar.xz  \
+            && sha256sum -c node-${arch}-sha256.txt
+RUN cd /usr/local && arch=`cat /etc/plat-arch`                        \
+                  && tar xJf /tmp/node-$node_ver-linux-${arch}.tar.xz \
+                  && ln -s node-$node_ver-linux-${arch} node          \
+                  && cd bin                                           \
+                  && ln -s ../node/bin/node node                      \
+                  && ln -s ../node/bin/npm  npm                       \
                   && ln -s ../node/bin/npx  npx            
 
 ENV GOSU_VERSION 1.14

diff --git a/docker/pdrangular/cacerts/README.md b/docker/pdrangular/cacerts/README.md
@@ -0,0 +1,13 @@
+This directory contains non-standard CA certificates needed to build the docker
+images. 
+
+Failures building the Docker containers defined in ../ due to SSL certificate
+verification errors may be a consequence of your local network's firewall.  In
+particular, the firewall may be substituting external site certificates with
+its own signed by a non-standard CA certficate (chain).  If so, you can place 
+the necessary certificates into this directory; they will be passed into the
+containers, allowing them to safely connect to those external sites.
+
+Be sure the certificates are in PEM format and include a .crt file extension.
+
+Do not remove this README file; doing so may cause a Docker build faiure.
diff --git a/docker/pdrangular/node-arm64-sha256.txt b/docker/pdrangular/node-arm64-sha256.txt
@@ -0,0 +1 @@
+73224b331d2bb0f05fd2ff62d1c736460f12f8e8757e8afbbcdf11a7f1464da5  node-v14.19.0-linux-arm64.tar.xz
diff --git a/docker/pdrangular/node-sha256.txt → docker/pdrangular/node-x64-sha256.txt b/docker/pdrangular/node-sha256.txt → docker/pdrangular/node-x64-sha256.txt
diff --git a/docker/run.sh b/docker/run.sh
@@ -76,6 +76,9 @@ while [ "$1" != "" ]; do
         -d|--docker-build)
             dodockbuild=1
             ;;
+        -D|--no-docker-build)
+            dodockbuild=0
+            ;;
         --dist-dir)
             shift
             distdir="$1"
@@ -172,7 +175,7 @@ if [ -z "$dodockbuild" ]; then
     fi
 fi
 
-[ -z "$dodockbuild" ] || {
+[ "$dodockbuild" != "1" ] || {
     echo '#' Building missing docker containers...
     $execdir/dockbuild.sh
 }

diff --git a/oar-build/_dockbuild.sh b/oar-build/_dockbuild.sh
@@ -60,6 +60,21 @@ function setup_build {
     BUILD_OPTS=`collect_build_opts`
 }
 
+function cp_ca_certs_to {
+    # assuming we are in the docker dir
+    [ \! -d cacerts ] || {
+        crts=`compgen -G 'cacerts/*.crt' || true`
+        [ -z "$crts" ] || {
+            echo "${prog}: installing CA certs from docker/cacerts"
+            for cont in $@; do
+                mkdir -p $cont/cacerts
+                echo '+' cp $crts cacerts/README.md $cont/cacerts
+                cp $crts cacerts/README.md $cont/cacerts
+            done
+        }
+    }
+}
+
 function help {
     helpfile=$OAR_BUILD_DIR/dockbuild_help.txt
     [ -f "$OAR_DOCKER_DIR/dockbuild_help.txt" ] && \

diff --git a/oar-metadata b/oar-metadata
diff --git a/python/tests/nistoar/pdr/preserv/data/simplesip/_nerdm.json b/python/tests/nistoar/pdr/preserv/data/simplesip/_nerdm.json
@@ -1,8 +1,8 @@
 {
     "@context": "https://data.nist.gov/od/dm/nerdm-pub-context.jsonld",
-    "_schema": "https://data.nist.gov/od/dm/nerdm-schema/v0.6#",
+    "_schema": "https://data.nist.gov/od/dm/nerdm-schema/v0.7#",
     "_extensionSchemas": [
-        "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/PublishedDataResource"
+        "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/PublishedDataResource"
     ],
     "@type": [
         "nrdp:PublishedDataResource"
@@ -58,7 +58,7 @@
             "refType": "IsReferencedBy",
             "location": "https://doi.org/10.1364/OE.24.014100",
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/bib/v0.6#/definitions/DCiteDocumentReference"
+                "https://data.nist.gov/od/dm/nerdm-schema/bib/v0.7#/definitions/DCiteDocumentReference"
             ]
         }
     ],
@@ -76,7 +76,7 @@
                 "dcat:Distribution"
             ],
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile"
+                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile"
             ]
         },
         {
@@ -89,7 +89,7 @@
                 "dcat:Distribution"
             ],
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile"
+                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile"
             ]
         },
         {
@@ -113,7 +113,7 @@
                 "dcat:Distribution"
             ],
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile"
+                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile"
             ]
         },
         {
@@ -126,7 +126,7 @@
                 "dcat:Distribution"
             ],
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile"
+                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile"
             ]
         },
         {
@@ -139,7 +139,7 @@
                 "dcat:Distribution"
             ],
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile"
+                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile"
             ]
         },
         {
@@ -152,7 +152,7 @@
                 "dcat:Distribution"
             ],
             "_extensionSchemas": [
-                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.6#/definitions/DataFile"
+                "https://data.nist.gov/od/dm/nerdm-schema/pub/v0.7#/definitions/DataFile"
             ]
         }
     ],

diff --git a/scripts/install_ca_certs.sh b/scripts/install_ca_certs.sh
@@ -0,0 +1,37 @@
+#! /bin/bash
+#
+# install_ca_certs.sh -- copy the specified CA certificates into this source so that they can be used
+#                        to build the software via docker.
+#
+# usage: install_ca_certs.sh CA_CERT_FILE...
+#
+#   where CA_CERT_FILE is a file path to a CA certificate to install
+#
+# This script helps address the problem with docker-based builds when run within a firewall that
+# replaces external site certificates with ones signed by a non-standard CA, causing the retrieval
+# of software dependencies to fail.  This script is used by oar-docker's localbuild script to receive 
+# extra CA certificates that addresses such failures.  Because localdeploy makes no assumptions about 
+# how this source code repository builds using docker, this script encapsulates that knowledge on 
+# behalf of localbuild.
+#
+# Note: if this repository does not require/support use of non-standard CA certificates, remove (or
+# rename) this script.
+#
+set -e
+prog=`basename $0`
+execdir=`dirname $0`
+[ "$execdir" = "" -o "$execdir" = "." ] && execdir=$PWD
+basedir=`dirname $execdir`
+
+cacertdir="$basedir/docker/cacerts"
+[ -d "$cacertdir" ] || exit 0         # I guess we don't need the certs
+
+crts=`echo $@ | sed -e 's/^ *//' -e 's/ *$//'`
+[ -n "$crts" ] || {
+    print "${prog}: Missing cert file argument"
+    false
+}
+
+echo '+' cp $crts $cacertdir
+cp $crts $cacertdir
+
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		73224b331d2bb0f05fd2ff62d1c736460f12f8e8757e8afbbcdf11a7f1464da5 node-v14.19.0-linux-arm64.tar.xz
+1 −0		.github/workflows/testall.yml
+3 −3		jq/pod2nerdm.jq
+12 −12		jq/tests/test_pod2nerdm.jqt
+9 −7		jq/tests/test_podds2resource.py