Skip to content

Commit

Permalink
Updated dockerfile, geo_coords
Browse files Browse the repository at this point in the history
  • Loading branch information
@avanetten
avanetten committed Jul 20, 2019
1 parent e68546a commit 025dee0
Show file tree
Hide file tree
Showing 7 changed files with 131 additions and 93 deletions.
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ During the test phase, input images of arbitrary size are processed.
--label_map_path class_labels_car.pbtxt \
--train_model_path [ssd_train_path] \
--tf_cfg_train_file ssd_inception_v2_simrdwn.config \
--use_tfrecords=1 \
--use_tfrecords=0 \
--testims_dir cowc/Utah_AGRC \
--keep_test_slices 0 \
--test_slice_sep __ \
Expand All @@ -142,7 +142,7 @@ During the test phase, input images of arbitrary size are processed.
# YOLT vehicle search
python /raid/local/src/simrdwn/core/simrdwn.py \
--framework yolt \
--framework yolt2 \
--mode test \
--outname dense_cowc \
--label_map_path class_labels_car.pbtxt \
Expand Down
127 changes: 74 additions & 53 deletions docker/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,6 @@
FROM nvidia/cuda:9.0-devel-ubuntu16.04
MAINTAINER avanetten

# nvidia-docker build -t simrdwn2.1 . # build (use existing packages)
# nvidia-docker build --no-cache -t simrdwn2.2 . # rebuild from scratch
# NV_GPU=0 nvidia-docker run -it -v /local_data:/local_data --name simrdwn2.2_gpu0 simrdwn2.2
# NV_GPU=0 nvidia-docker run -it -v /local_data:/local_data -v /cosmiq:/cosmiq --name simrdwn2.1_gpu0 simrdwn2.1

# IF YOU WANT PROGRESS PRINTED TO TERMINAL
# Update model_main to log to screen...
# https://stackoverflow.com/questions/52016255/tensorflow-object-detection-api-not-displaying-global-steps
Expand Down Expand Up @@ -73,78 +68,84 @@ SHELL ["/bin/bash", "-c"]
ENV PATH /opt/conda/bin:$PATH

# install anaconda
RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O ~/miniconda.sh && \
# https://repo.continuum.io/miniconda/
RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \
/bin/bash ~/miniconda.sh -b -p /opt/conda && \
rm ~/miniconda.sh && \
/opt/conda/bin/conda clean -tipsy && \
ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
echo "conda activate base" >> ~/.bashrc
ENV TINI_VERSION v0.16.1
# ENV TINI_VERSION v0.16.1
ENV TINI_VERSION v0.18.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
RUN chmod +x /usr/bin/tini

# use conda-forge instead of default channel
RUN conda update conda && \
conda config --remove channels defaults && \
conda config --add channels conda-forge
RUN conda config --add channels conda-forge
# && \
# conda update conda
# conda config --remove channels defaults && \

# set up conda environment and add to $PATH
RUN conda create -n simrdwn2 python=3.6 \
&& echo "source activate simrdwn2" > ~/.bashrc
ENV PATH /opt/conda/envs/simrdwn2/bin:$PATH
RUN conda create -n simrdwn python=3.6 \
&& echo "source activate simrdwn" > ~/.bashrc
ENV PATH /opt/conda/envs/simrdwn/bin:$PATH

# install GPU version of tensorflow
RUN source activate simrdwn2 && \
conda install -n simrdwn2 -c defaults tensorflow-gpu=1.13.1
RUN source activate simrdwn && \
conda install -n simrdwn -c defaults tensorflow-gpu=1.13.1

# install keras with tf backend
ENV KERAS_BACKEND=tensorflow
RUN source activate simrdwn2 \
&& conda install -n simrdwn2 keras

RUN conda install -n simrdwn2 \
#awscli \
affine \
pyproj \
RUN source activate simrdwn \
&& conda install -n simrdwn keras=2.2.4

RUN source activate simrdwn \
&& conda install -n simrdwn \
libgdal=2.4.1 \
gdal=2.4.1 \
geopandas=0.5.0 \
opencv=4.1.0 \
affine=2.2.2 \
pyproj=2.1.3 \
pyhamcrest=1.9.0 \
cython \
contextlib2 \
fiona \
h5py \
ncurses \
jupyter \
jupyterlab \
ipykernel \
libgdal \
matplotlib \
ncurses \
numpy \
#opencv=3.4.1 \
#py-opencv \
pandas \
pillow \
pip \
scipy \
scikit-image \
scikit-learn \
shapely \
gdal \
rtree \
testpath \
tqdm \
pandas \
geopandas \
rasterio \
opencv=4.0.0 \
cython=0.29.10 \
contextlib2=0.5.5 \
h5py=2.9.0 \
ncurses=6.1 \
jupyter=1.0.0 \
jupyterlab=0.35.6 \
ipykernel=5.1.1 \
matplotlib=3.1.0 \
numpy=1.16.4 \
pandas=0.24.2 \
pillow=6.0.0 \
pip=19.1.1 \
scipy=1.3.0 \
scikit-image=0.15.0 \
scikit-learn=0.21.2 \
shapely=1.6.4 \
rtree=0.8.3 \
testpath=0.4.2 \
tqdm=4.32.1 \
statsmodels=0.10.1 \
&& conda clean -p \
&& conda clean -t \
&& conda clean --yes --all

RUN pip install statsmodels
# Below is a hack to get around conda issues (2019-07-19)
RUN source activate simrdwn \
&& conda install -n simrdwn \
-c conda-forge rasterio=1.0.24 \
&& conda clean -p \
&& conda clean -t \
&& conda clean --yes --all

# tf object detection api
RUN git clone https://github.com/tensorflow/models.git /tensorflow/models
WORKDIR /tensorflow/models/research/
RUN git checkout r1.13.0
#WORKDIR /tensorflow/models/research/
#RUN protoc object_detection/protos/*.proto --python_out=.
# WORKDIR /tensorflow/models/research/
Expand All @@ -167,8 +168,28 @@ RUN export PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim
ENV PYTHONPATH $PYTHONPATH:/tensorflow/models/:/tensorflow/models/slim
ENV PYTHONPATH $PYTHONPATH:/tensorflow/models/:/tensorflow/models/research/slim

# # this always seems to fail, so do it manually (as above)
# RUN git clone https://github.com/cocodataset/cocoapi.git /cocoapi
# WORKDIR /cocoapi/PythonAPI
# RUN make
# RUN cp -r pycocotools /tensorflow/models/research/
# # From tensorflow/models/research/
# WORKDIR /tensorflow/models/research/
# RUN protoc object_detection/protos/*.proto --python_out=.

# # test
# WORKDIR /tensorflow/models/research/
# RUN PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim
# RUN python /tensorflow/models/research/object_detection/builders/model_builder_test.py
#
# # RUN source activate simrdwn && \
# # python /tensorflow/models/research/object_detection/builders/model_builder_test.py
# # # ENV MODELS_DIR /tensorflow/models/research/
# # # PYTHONPATH=${MODELS_DIR}/research:$MODELS_DIR/research/slim \
# # #PYTHON $MODELS_DIR/research/object_detection/builders/model_builder_test.py

# # add a jupyter kernel for the conda environment in case it's wanted
RUN source activate simrdwn2 && python -m ipykernel.kernelspec
RUN source activate simrdwn && python -m ipykernel.kernelspec
RUN python -m ipykernel.kernelspec


Expand Down
28 changes: 19 additions & 9 deletions simrdwn/core/add_geo_coords.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,12 @@ def add_geo_coords_to_df(df_, inProj_str='epsg:4326', outProj_str='epsg:3857',
# raster geo transform
gdal_geomTransform = gdal.Open(raster_loc).GetGeoTransform()
affineObject = rio.open(raster_loc).transform

if verbose:
print("raster_loc:", raster_loc)
print("gdal_geomTransform:", gdal_geomTransform)
print("gdal_geomTransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0))",
gdal_geomTransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0))
print("affineObject:", affineObject)
geo_dict[raster_loc] = [gdal_geomTransform, affineObject]

Expand All @@ -204,14 +207,21 @@ def add_geo_coords_to_df(df_, inProj_str='epsg:4326', outProj_str='epsg:3857',
row['Xmin_Glob'], row['Ymin_Glob'],
row['Xmax_Glob'], row['Ymax_Glob'])

out_arr_row, poly_geo = get_row_geo_coords(
row, affineObject=affineObject,
gdal_geomTransform=gdal_geomTransform,
inProj_str=inProj_str, outProj_str=outProj_str,
verbose=verbose)
out_arr.append(out_arr_row)
if create_geojson:
out_arr_json.append(poly_geo)
# check if the the image as geographic metadata
if gdal_geomTransform == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0):
out_arr_row = 8 * [0]
out_arr.append(out_arr_row)

# else, get geo locations
else:
out_arr_row, poly_geo = get_row_geo_coords(
row, affineObject=affineObject,
gdal_geomTransform=gdal_geomTransform,
inProj_str=inProj_str, outProj_str=outProj_str,
verbose=verbose)
out_arr.append(out_arr_row)
if create_geojson:
out_arr_json.append(poly_geo)

# update dataframe
# [lon0, lat0, lon1, lat1, x0_wmp, y0_wmp, x1_wmp, y1_wmp]
Expand All @@ -227,7 +237,7 @@ def add_geo_coords_to_df(df_, inProj_str='epsg:4326', outProj_str='epsg:3857',

# geodataframe if desired
# https://gis.stackexchange.com/questions/174159/convert-a-pandas-dataframe-to-a-geodataframe
if create_geojson:
if create_geojson and (len(out_arr_json) > 0):
crs_init = {'init': inProj_str}
df_json = pd.DataFrame(out_arr_json, columns=['geometry'])
# add important columns to df_json
Expand Down
23 changes: 16 additions & 7 deletions simrdwn/core/infer_detections.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@
tf.flags.DEFINE_string('inference_graph', None,
'Path to the inference graph with embedded weights.')
tf.flags.DEFINE_boolean('verbose', False, 'Lots o print statements')
tf.flags.DEFINE_boolean('use_tfrecords', True, 'Switch to use tfrecords')

tf.flags.DEFINE_boolean('use_tfrecords', False, 'Switch to use tfrecords')

# tfrecords
tf.flags.DEFINE_string('input_tfrecord_paths', None,
Expand Down Expand Up @@ -97,8 +96,8 @@
tf.flags.DEFINE_integer('BGR2RGB', 0,
'Sometimes we need to change cv2 images to BGR')


FLAGS = tf.flags.FLAGS
# print("FLAGS:", FLAGS)


###############################################################################
Expand Down Expand Up @@ -166,7 +165,7 @@ def main(_):
str(t1 - t0) + ' seconds')

else:

# infer on csv
if FLAGS.verbose:
print("min_thresh:", FLAGS.min_thresh)
t0 = time.time()
Expand All @@ -180,7 +179,7 @@ def main(_):
tf.import_graph_def(graph_def, name='')
print("Time to load graph:", time.time() - t0, "seconds")

with open(FLAGS.output_csv_path, 'wb') as csvfile:
with open(FLAGS.output_csv_path, 'w') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',')
output_columns = ['Loc_Tmp', u'Prob', u'Xmin',
u'Ymin', u'Xmax', u'Ymax', u'Category']
Expand All @@ -196,9 +195,19 @@ def main(_):

line_count = 0
for i, image_path in enumerate(image_paths):

image_root = os.path.basename(image_path).strip()
image_bgr = cv2.imread(image_path.strip(), 1)
# print("image_path:", image_path)
# print("str image_path.strip", str(image_path.strip()))
# print("image_path.strip", image_path.strip().decode('utf-8'))
#print("str image_path.strip", str(image_path.strip().decode('utf-8')))
# print("os.path.exists image_path.strip", os.path.exists(image_path.strip()))
# print("os.path.exists str image_path.strip", os.path.exists(str(image_path.strip())))
# print("os.path.exists str image_path.strip", os.path.exists(str(image_path.strip()).strip()))
# print("os.path.exists str image_path.strip", os.path.exists(str(image_path.strip().decode('utf-8'))))

# image_bgr = cv2.imread(image_path, cv2.IMREAD_COLOR)
# image_bgr = cv2.imread(str(image_path.strip()), 1)
image_bgr = cv2.imread(str(image_path.strip().decode('utf-8')), 1)
# invert colors, if required
if FLAGS.BGR2RGB == 1:
image = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
Expand Down
2 changes: 2 additions & 0 deletions simrdwn/core/post_process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,8 @@ def refine_df(df, groupby='Image_Path',
if verbose:
print("num boxes_all:", len(xmins))
print("num good_idxs:", len(good_idxs))
if len(boxes) == 0:
print("Error, No boxes detected!")
boxes = boxes[good_idxs]
scores = scores[good_idxs]
df_idxs = df_idxs[good_idxs]
Expand Down
Loading

0 comments on commit 025dee0

Please sign in to comment.