Skip to content

Commit

Permalink
fix: deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
mhf-ir committed Dec 6, 2022
1 parent c40d876 commit 94d64f3
Show file tree
Hide file tree
Showing 60 changed files with 1,936 additions and 1,391 deletions.
1 change: 1 addition & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ RUN apk --no-cache add ca-certificates \

ENV ASM_ANALYTICS_COLLECTOR_MMDB_ASN_PATH="/GeoLite2-ASN.mmdb" \
ASM_ANALYTICS_COLLECTOR_MMDB_CITY_PATH="/GeoLite2-City.mmdb" \
ASM_ANALYTICS_COLLECTOR_REDIS_URI="redis://analytics-redis:6379/0" \
ASM_ANALYTICS_COLLECTOR_POSTGIS_URI="postgres://geonames:geonames@analytics-postgis:5432/geonames"

ADD tmp/GeoLite2-ASN.mmdb /GeoLite2-ASN.mmdb
Expand Down
10 changes: 4 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@
# prepare dependencies
./make.sh
docker-compose -f docker-compose.dev.yml up -d
# import clickhouse schema
docker exec -it analytics-clickhouse-client bash
clickhouse-client --multiquery --host analytics-clickhouse-single --user 'analytics' --password 'password123123' < /tmp/schema.sql

# run cli
docker exec -it analytics-clickhouse-client /usr/bin/clickhouse-client --vertical --host analytics-clickhouse-single --user 'analytics' --password 'password123123'
# update schema
docker exec -it analytics-clickhouse bash -c 'clickhouse-client --multiquery < /schema.sql'
# access the console of clickhouse
docker exec -it analytics-clickhouse bash -c 'clickhouse-client --vertical'
```

<div>
Expand Down
8 changes: 4 additions & 4 deletions cert/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -
## Cluster

```bash
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch01.analytics-clickhouse.net.private:8443
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch01.analytics-clickhouse.net.private:9440
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch01.analytics-clickhouse.net.private:9010
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch01.analytics-clickhouse.net.private:9281
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch1.analytics-clickhouse.net.private:8443
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch1.analytics-clickhouse.net.private:9440
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch1.analytics-clickhouse.net.private:9010
openssl s_client -CAfile ca.pem -cert client-fullchain.pem -key client-key.pem -connect ch1.analytics-clickhouse.net.private:9281
```
7 changes: 1 addition & 6 deletions cert/csr-client.json
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
{
"CN": "clickhouse client",
"hosts": [
"127.0.0.1",
"localhost",
"analytics-clickhouse",
"*.analytics-clickhouse.net.private"
],
"hosts": ["127.0.0.1", "localhost", "*.analytics-clickhouse.net.private"],
"key": {
"algo": "ecdsa",
"size": 256
Expand Down
7 changes: 1 addition & 6 deletions cert/csr-server.json
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
{
"CN": "clickhouse private node",
"hosts": [
"127.0.0.1",
"localhost",
"analytics-clickhouse",
"*.analytics-clickhouse.net.private"
],
"hosts": ["127.0.0.1", "localhost", "*.analytics-clickhouse.net.private"],
"key": {
"algo": "ecdsa",
"size": 256
Expand Down
6 changes: 3 additions & 3 deletions clickhouse-cluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ Using following steps:
Copy each node data to your desire path of server:

```bash
scp -r ready/ch01 [email protected]:/root/
scp -r ready/ch02 [email protected]:/root/
scp -r ready/ch03 [email protected]:/root/
scp -r ready/ch1 [email protected]:/root/
scp -r ready/ch2 [email protected]:/root/
scp -r ready/ch3 [email protected]:/root/
```

## Running clickhouse nodes
Expand Down
89 changes: 45 additions & 44 deletions clickhouse-cluster/init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ PROJECT_DIR="$( realpath -e -- "$CURRENT_DIR/../"; )";
ASM_CH_NODE1_IP=$1
ASM_CH_NODE2_IP=$2
ASM_CH_NODE3_IP=$3
ASM_CH_APP_IP=$4
ASM_CH_MANGEGMENT_IP=$4
ASM_COLLECTOR_DOMAIN=$5

if [ -z "$ASM_COLLECTOR_DOMAIN" ]; then
Expand All @@ -35,61 +35,62 @@ fi
if validate_ip $ASM_CH_NODE1_IP; then NODE01_IP_VALID='1'; else NODE01_IP_VALID='0'; fi
if validate_ip $ASM_CH_NODE2_IP; then NODE02_IP_VALID='1'; else NODE02_IP_VALID='0'; fi
if validate_ip $ASM_CH_NODE3_IP; then NODE03_IP_VALID='1'; else NODE03_IP_VALID='0'; fi
if validate_ip $ASM_CH_APP_IP; then APP_IP_VALID='1'; else APP_IP_VALID='0'; fi
if validate_ip $ASM_CH_MANGEGMENT_IP; then MANGEGMENT_IP_VALID='1'; else MANGEGMENT_IP_VALID='0'; fi

if [[ $NODE01_IP_VALID == "0" ]]; then echo "invalid node 1 IP $ASM_CH_NODE1_IP"; exit 1; fi
if [[ $NODE02_IP_VALID == "0" ]]; then echo "invalid node 2 IP $ASM_CH_NODE2_IP"; exit 1; fi
if [[ $NODE03_IP_VALID == "0" ]]; then echo "invalid node 3 IP $ASM_CH_NODE3_IP"; exit 1; fi
if [[ $APP_IP_VALID == "0" ]]; then echo "invalid app IP $APP_IP_VALID"; exit 1; fi
if [[ $MANGEGMENT_IP_VALID == "0" ]]; then echo "invalid mangemment IP $ASM_CH_MANGEGMENT_IP"; exit 1; fi

echo "All nodes IP seems fine:"
echo "Node 1 IP: $ASM_CH_NODE1_IP"
echo "Node 2 IP: $ASM_CH_NODE2_IP"
echo "Node 3 IP: $ASM_CH_NODE3_IP"
echo "Node 1 IP: $ASM_CH_NODE1_IP"
echo "Node 2 IP: $ASM_CH_NODE2_IP"
echo "Node 3 IP: $ASM_CH_NODE3_IP"
echo "Magegment IP: $ASM_CH_MANGEGMENT_IP"

RANDOM_PASSWORD=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 32)

for i in $(seq 1 3); do
declare "NODE_PATH"=$CURRENT_DIR/ready/ch0$i
NODE_PATH=$CURRENT_DIR/ready/srv$1/aasaam-analytics
mkdir -p $NODE_PATH
cp -rf $CURRENT_DIR/template/* $NODE_PATH/
cp -f $PROJECT_DIR/cert/{ca.pem,dhparam.pem,client-fullchain.pem,client-key.pem,server-fullchain.pem,server-key.pem} $NODE_PATH/cert/
cp -rf $CURRENT_DIR/template/.env $NODE_PATH/.env
cp -rf $CURRENT_DIR/template/get-public-cert.sh $NODE_PATH/get-public-cert.sh

sed -i "s+__RANDOM_PASSWORD__+$RANDOM_PASSWORD+g" $NODE_PATH/.env

sed -i "s+__ASM_CH_NODE1_IP__+$ASM_CH_NODE1_IP+g" $NODE_PATH/.env
sed -i "s+__ASM_CH_NODE2_IP__+$ASM_CH_NODE2_IP+g" $NODE_PATH/.env
sed -i "s+__ASM_CH_NODE3_IP__+$ASM_CH_NODE3_IP+g" $NODE_PATH/.env

sed -i "s+__ASM_COLLECTOR_DOMAIN__+$ASM_COLLECTOR_DOMAIN+g" $NODE_PATH/.env

rm -rf $NODE_PATH/nginx-exposer/acl.conf
touch $NODE_PATH/nginx-exposer/acl.conf
echo "allow $ASM_CH_NODE1_IP;" >> $NODE_PATH/nginx-exposer/acl.conf
echo "allow $ASM_CH_NODE2_IP;" >> $NODE_PATH/nginx-exposer/acl.conf
echo "allow $ASM_CH_NODE3_IP;" >> $NODE_PATH/nginx-exposer/acl.conf
echo "allow $ASM_CH_APP_IP;" >> $NODE_PATH/nginx-exposer/acl.conf

sed -i "s+__ASM_COLLECTOR_DOMAIN__+$ASM_COLLECTOR_DOMAIN+g" $NODE_PATH/get-public-cert.sh
chmod 500 $NODE_PATH/get-public-cert.sh

rm -rf $NODE_PATH/tmp

__NODE_ID__=$i
__OTHER_NODE_1__="2"
__OTHER_NODE_2__="3"
if [[ $i == "2" ]]; then
__OTHER_NODE_1__="1"
__OTHER_NODE_2__="3"
elif [[ $i == "3" ]]; then
__OTHER_NODE_1__="2"
__OTHER_NODE_2__="1"
fi
# cp -rf $CURRENT_DIR/template/.env $NODE_PATH/.env
# cp -rf $CURRENT_DIR/template/get-public-cert.sh $NODE_PATH/get-public-cert.sh

# sed -i "s+__RANDOM_PASSWORD__+$RANDOM_PASSWORD+g" $NODE_PATH/.env

# sed -i "s+__ASM_CH_NODE1_IP__+$ASM_CH_NODE1_IP+g" $NODE_PATH/.env
# sed -i "s+__ASM_CH_NODE2_IP__+$ASM_CH_NODE2_IP+g" $NODE_PATH/.env
# sed -i "s+__ASM_CH_NODE3_IP__+$ASM_CH_NODE3_IP+g" $NODE_PATH/.env

# sed -i "s+__ASM_COLLECTOR_DOMAIN__+$ASM_COLLECTOR_DOMAIN+g" $NODE_PATH/.env

# rm -rf $NODE_PATH/nginx-exposer/acl.conf
# touch $NODE_PATH/nginx-exposer/acl.conf
# echo "allow $ASM_CH_NODE1_IP;" >> $NODE_PATH/nginx-exposer/acl.conf
# echo "allow $ASM_CH_NODE2_IP;" >> $NODE_PATH/nginx-exposer/acl.conf
# echo "allow $ASM_CH_NODE3_IP;" >> $NODE_PATH/nginx-exposer/acl.conf
# echo "allow $ASM_CH_APP_IP;" >> $NODE_PATH/nginx-exposer/acl.conf

# sed -i "s+__ASM_COLLECTOR_DOMAIN__+$ASM_COLLECTOR_DOMAIN+g" $NODE_PATH/get-public-cert.sh
# chmod 500 $NODE_PATH/get-public-cert.sh

# rm -rf $NODE_PATH/tmp

# __NODE_ID__=$i
# __OTHER_NODE_1__="2"
# __OTHER_NODE_2__="3"
# if [[ $i == "2" ]]; then
# __OTHER_NODE_1__="1"
# __OTHER_NODE_2__="3"
# elif [[ $i == "3" ]]; then
# __OTHER_NODE_1__="2"
# __OTHER_NODE_2__="1"
# fi

sed -i "s+__NODE_ID__+$__NODE_ID__+g" $NODE_PATH/.env
sed -i "s+__OTHER_NODE_1__+$__OTHER_NODE_1__+g" $NODE_PATH/.env
sed -i "s+__OTHER_NODE_2__+$__OTHER_NODE_2__+g" $NODE_PATH/.env
# sed -i "s+__NODE_ID__+$__NODE_ID__+g" $NODE_PATH/.env
# sed -i "s+__OTHER_NODE_1__+$__OTHER_NODE_1__+g" $NODE_PATH/.env
# sed -i "s+__OTHER_NODE_2__+$__OTHER_NODE_2__+g" $NODE_PATH/.env

done
# done
12 changes: 6 additions & 6 deletions clickhouse-cluster/template/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ services:
hard: 262144

extra_hosts:
- "ch01.${ASM_CH_DOMAIN}:${ASM_CH_NODE1_IP}"
- "ch02.${ASM_CH_DOMAIN}:${ASM_CH_NODE2_IP}"
- "ch03.${ASM_CH_DOMAIN}:${ASM_CH_NODE3_IP}"
- "ch1.${ASM_CH_DOMAIN}:${ASM_CH_NODE1_IP}"
- "ch2.${ASM_CH_DOMAIN}:${ASM_CH_NODE2_IP}"
- "ch3.${ASM_CH_DOMAIN}:${ASM_CH_NODE3_IP}"

logging:
driver: "journald"
Expand Down Expand Up @@ -106,9 +106,9 @@ services:
- ./.env

extra_hosts:
- "ch01.${ASM_CH_DOMAIN}:${ASM_CH_NODE1_IP}"
- "ch02.${ASM_CH_DOMAIN}:${ASM_CH_NODE2_IP}"
- "ch03.${ASM_CH_DOMAIN}:${ASM_CH_NODE3_IP}"
- "ch1.${ASM_CH_DOMAIN}:${ASM_CH_NODE1_IP}"
- "ch2.${ASM_CH_DOMAIN}:${ASM_CH_NODE2_IP}"
- "ch3.${ASM_CH_DOMAIN}:${ASM_CH_NODE3_IP}"

logging:
driver: "journald"
Expand Down
106 changes: 106 additions & 0 deletions clickhouse-cluster/template/mangement/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
version: "3"

services:
# backend
analytic-backend:
image: ghcr.io/aasaam/analytic-backend:latest
depends_on:
- analytic-redis
- analytic-rest-captcha
- analytic-postgres
container_name: analytic-backend
volumes:
- ./cert/ca.pem:/app/api/clickhouse-cert/ca.pem:ro
- ./cert/client-fullchain.pem:/app/api/clickhouse-cert/client-fullchain.pem:ro
- ./cert/client-key.pem:/app/api/clickhouse-cert/client-key.pem:ro

env_file:
- ./.env

extra_hosts:
- "ch1.${ASM_CH_DOMAIN}:${ASM_CH_NODE1_IP}"
- "ch2.${ASM_CH_DOMAIN}:${ASM_CH_NODE2_IP}"
- "ch3.${ASM_CH_DOMAIN}:${ASM_CH_NODE3_IP}"

# front and nginx
analytic-web:
image: ghcr.io/aasaam/analytic-web:latest
depends_on:
- analytic-backend
container_name: analytic-web
ports:
- 80:80
- 443:443
environment:
ASM_NGINX_WORKER_PROCESSES: ${ASM_NGINX_WORKER_PROCESSES:-2}
ASM_NGINX_WORKER_RLIMIT_NOFILE: ${ASM_NGINX_WORKER_RLIMIT_NOFILE:-1024}
ASM_NGINX_WORKER_CONNECTIONS: ${ASM_NGINX_WORKER_CONNECTIONS:-512}
ASM_NGINX_ERROR_LOG_LEVEL: ${ASM_NGINX_ERROR_LOG_LEVEL:-warn}
ASM_NGINX_TMPFS_CACHE_SIZE_MB: ${ASM_NGINX_TMPFS_CACHE_SIZE_MB:-128}
ASM_NGINX_LARGE_CACHE_SIZE_MB: ${ASM_NGINX_LARGE_CACHE_SIZE_MB:-2048}
ASM_NGINX_LIMIT_CONNECTION: ${ASM_NGINX_LIMIT_CONNECTION:-16}
ASM_NGINX_LIMIT_RPS: ${ASM_NGINX_LIMIT_RPS:-4}
ASM_NGINX_LIMIT_RATE_KBS: ${ASM_NGINX_LIMIT_RATE_KBS:-32}
ASM_NGINX_LIMIT_BURST: ${ASM_NGINX_LIMIT_BURST:-64}
ASM_NGINX_LIMIT_RATE_AFTER_MB: ${ASM_NGINX_LIMIT_RATE_AFTER_MB:-1}
ASM_APP_PORT: ${ASM_APP_PORT:-3001}
ASM_PUBLIC_BASE_URL: ${ASM_PUBLIC_BASE_URL:-/}
ASM_FRONT_CP_PORT: ${ASM_FRONT_CP_PORT:-3000}
ASM_FRONT_CP_PATH: ${ASM_FRONT_CP_PATH:-_cp}
ASM_ACCESS_LOG_COMMENT: ${ASM_ACCESS_LOG_COMMENT:-}
ASM_PUBLIC_APP_LANG: ${ASM_PUBLIC_APP_LANG:-en}
tmpfs:
- /nginx-tmpfs:rw,nodev,nosuid,noexec,noatime,size=${ASM_NGINX_TMPFS_CACHE_SIZE_MB:-192}m
volumes:
- ./default.conf.template:/etc/nginx/templates/default.conf.template
- ./cert/fullchain.pem:/cert/fullchan.pem:ro
- ./cert/privkey.pem:/cert/privkey.pem:ro
- ./cert/chain.pem:/cert/ca.pem:ro
- ./cert/dhparam.pem:/cert/dhparam.pem:ro

# databases
analytic-redis:
container_name: analytic-redis
image: redis:6

analytic-rest-captcha:
container_name: analytic-captcha
image: ghcr.io/aasaam/rest-captcha:latest
command: run -return-value

analytic-postgres:
container_name: analytic-postgres
environment:
POSTGRES_DB: ${ASM_POSTGRES_DB:-pg-db}
POSTGRES_USER: ${ASM_POSTGRES_USER:-pg-user}
POSTGRES_PASSWORD: ${ASM_POSTGRES_PASSWORD:-pg-password}
POSTGRES_HOST_AUTH_METHOD: trust
image: postgres:13
volumes:
- postgres:/var/lib/postgresql/data

analytic-grafana:
container_name: analytic-grafana
image: ghcr.io/aasaam/grafana:latest

environment:
- GF_SERVER_ROOT_URL=https://grafana.analyt.ir/grafana/
- GF_SERVER_SERVE_FROM_SUB_PATH=true

volumes:
- grafana:/var/lib/grafana

- ./cert/ca.pem:/cert/ca.pem:ro
- ./cert/client-fullchain.pem:/cert/client-fullchain.pem:ro
- ./cert/client-key.pem:/cert/client-key.pem:ro

extra_hosts:
- "ch1.${ASM_CH_DOMAIN}:${ASM_CH_NODE1_IP}"
- "ch2.${ASM_CH_DOMAIN}:${ASM_CH_NODE2_IP}"
- "ch3.${ASM_CH_DOMAIN}:${ASM_CH_NODE3_IP}"

volumes:
postgres:
driver: local
grafana:
driver: local
Loading

0 comments on commit 94d64f3

Please sign in to comment.