diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml new file mode 100644 index 000000000..d4f25e5f8 --- /dev/null +++ b/.github/workflows/github-ci.yml @@ -0,0 +1,27 @@ +on: + push: + branches-ignore: + - 'master' + pull_request: + branches: + - 'master' +jobs: + test_and_package: + runs-on: ubuntu-latest + continue-on-error: false + steps: + - name: Checking out code base + uses: actions/checkout@v2 + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: Pull ryuk + run: docker pull quay.io/testcontainers/ryuk:0.2.2 + continue-on-error: false + + - name: Verification + run: mvn clean verify -Dmaven.test.failure.ignore=false + continue-on-error: false diff --git a/.gitignore b/.gitignore index 7a6589d59..29fbc3916 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ .DS_Store target/* config/test.yml +gauth-creds-env.sh +idman-creds-env.sh diff --git a/Dockerfile b/Dockerfile index d4d158a86..522915424 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Santanu Sinha RUN apt-get clean && apt-get update && apt-get install -y --no-install-recommends software-properties-common RUN add-apt-repository ppa:openjdk-r/ppa && apt-get update -RUN apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates && apt-get install -y --no-install-recommends ca-certificates-java bash curl tzdata iproute2 zip unzip wget +RUN apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates ca-certificates-java bash curl tzdata iproute2 zip unzip wget EXPOSE 17000 @@ -12,9 +12,10 @@ EXPOSE 5701 VOLUME /var/log/foxtrot-server -ADD config/docker.yml docker.yml -ADD foxtrot-server/target/foxtrot*.jar server.jar +ADD config/docker* config/ +ADD foxtrot-server/target/foxtrot-server*.jar server.jar ADD scripts/local_es_setup.sh local_es_setup.sh +ADD startup.sh startup.sh -CMD sh -c "sleep 15 ; java -Dfile.encoding=utf-8 -XX:+${GC_ALGO-UseG1GC} -Xms${JAVA_PROCESS_MIN_HEAP-1g} -Xmx${JAVA_PROCESS_MAX_HEAP-1g} ${JAVA_OPTS} -jar server.jar server docker.yml" +CMD ./startup.sh diff --git a/GAUTH b/GAUTH new file mode 100644 index 000000000..9fb9c9b3e --- /dev/null +++ b/GAUTH @@ -0,0 +1,3 @@ + 280204865700-2hlhc09kfd7s5efek730pq515cn1q3p9.apps.googleusercontent.com + + 6dy_KawiZLdaX--SOH2xX4_y diff --git a/README.md b/README.md index 1b5f2fe2a..7c5610cf0 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,15 @@ Check the [Wiki](https://github.com/Flipkart/foxtrot/wiki/Introduction) for deta Version ---- +6.3.1-9 -0.1 +Docker +------ +Docker can be found on [DockerHub](https://hub.docker.com/layers/santanusinha/foxtrot/6.3.1-9/images/sha256-01cb327eb0353d31874681ee9ece4df8993b2152dfd3fa7279a31e3d7e32ee7e?context=explore) + +Docker can be customised using environment variables. Refer [here](https://github.com/Flipkart/foxtrot/blob/simple_auth/config/docker.yml) for the variables that need to be passed. + +Volume mount the config file in a docker. And pass in the full path to file `CONFIG_PATH` environment variable. Tech ----------- diff --git a/build.properties b/build.properties index c1fdb92cc..d440758b0 100644 --- a/build.properties +++ b/build.properties @@ -1 +1 @@ -BUILD_PROJECT_VERSION=6.3.1-8 \ No newline at end of file +BUILD_PROJECT_VERSION=6.3.1-9 \ No newline at end of file diff --git a/config/docker-gauth.yml b/config/docker-gauth.yml new file mode 100755 index 000000000..1d506d94d --- /dev/null +++ b/config/docker-gauth.yml @@ -0,0 +1,89 @@ +server: + applicationConnectors: + - type: http + port: 17000 + adminConnectors: + - type: http + port: 17001 + +swaggerScheme: ${SWAGGER_SCHEME} + +elasticsearch: + hosts: + - ${ELASTICSEARCH_HOST} + cluster: ${ELASTICSEARCH_CLUSTER_NAME} + tableNamePrefix: ${ELASTICSEARCH_TABLE_NAME_PREFIX} + port: ${ELASTICSEARCH_PORT} + connectionType: ${ELASTICSEARCH_PROTOCOL} + +hbase: + secure : false + tableName: ${HBASE_TABLE_NAME} + hbaseZookeeperQuorum: ${HBASE_ZOOKEEPER_QUORUM} + hbaseZookeeperClientPort: ${HBASE_ZOOKEEPER_CLIENT_PORT} + +cluster: + name: foxtrot + discovery: + type: ${DISCOVERY_TYPE} + disableMulticast: true + members: ["localhost:5701"] + +cacheConfig: + maxIdleSeconds: 15 + timeToLiveSeconds: 15 + +logging: + level: INFO + loggers: + com.flipkart.foxtrot.core.querystore.actions: DEBUG + org.apache.hadoop.hbase.zookeeper: WARN + org.apache.zookeeper: WARN + org.apache.hadoop.hbase.client: WARN + appenders: + - type: console + threshold: TRACE + timeZone: IST + logFormat: "%(%-5level) [%date] %X{TRACE-ID} [%thread] [%logger{0}]: %message%n" + +cardinality: + enabled: true + batchSize: 10 + active: true + interval: 86400 + initialDelay: 6 + maxCardinality: 20000 + +deletionconfig: + active: true + interval: 86400 + initialdelay: 60 + +esIndexOptimizationConfig: + active: true + interval: 86400 + initialDelay: 6 + +consoleHistoryConfig: + active: true + interval: 86400 + initialDelay: 10 + +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + +auth: + enabled: true + jwt: + issuerId: foxtrot-server + privateKey: ${PRIVATE_KEY} + sessionDuration: 15d + + provider: + type: OAUTH_GOOGLE + clientId: ${GOOGLE_CLIENT_ID} + clientSecret: ${GOOGLE_CLIENT_SECRET} + server: ${GOOGLE_CALLBACK_HOST_PORT} + secureEndpoint: ${GOOGLE_SECURE_ENDPOINT} \ No newline at end of file diff --git a/config/docker-idman.yml b/config/docker-idman.yml new file mode 100755 index 000000000..f3ad611d8 --- /dev/null +++ b/config/docker-idman.yml @@ -0,0 +1,89 @@ +server: + applicationConnectors: + - type: http + port: 17000 + adminConnectors: + - type: http + port: 17001 + +swaggerScheme: ${SWAGGER_SCHEME} + +elasticsearch: + hosts: + - ${ELASTICSEARCH_HOST} + cluster: ${ELASTICSEARCH_CLUSTER_NAME} + tableNamePrefix: ${ELASTICSEARCH_TABLE_NAME_PREFIX} + port: ${ELASTICSEARCH_PORT} + connectionType: ${ELASTICSEARCH_PROTOCOL} + +hbase: + secure : false + tableName: ${HBASE_TABLE_NAME} + hbaseZookeeperQuorum: ${HBASE_ZOOKEEPER_QUORUM} + hbaseZookeeperClientPort: ${HBASE_ZOOKEEPER_CLIENT_PORT} + +cluster: + name: foxtrot + discovery: + type: ${DISCOVERY_TYPE} + disableMulticast: true + members: ["localhost:5701"] + +cacheConfig: + maxIdleSeconds: 15 + timeToLiveSeconds: 15 + +logging: + level: INFO + loggers: + com.flipkart.foxtrot.core.querystore.actions: DEBUG + org.apache.hadoop.hbase.zookeeper: WARN + org.apache.zookeeper: WARN + org.apache.hadoop.hbase.client: WARN + appenders: + - type: console + threshold: TRACE + timeZone: IST + logFormat: "%(%-5level) [%date] %X{TRACE-ID} [%thread] [%logger{0}]: %message%n" + +cardinality: + enabled: true + batchSize: 10 + active: true + interval: 86400 + initialDelay: 6 + maxCardinality: 20000 + +deletionconfig: + active: true + interval: 86400 + initialdelay: 60 + +esIndexOptimizationConfig: + active: true + interval: 86400 + initialDelay: 6 + +consoleHistoryConfig: + active: true + interval: 86400 + initialDelay: 10 + +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + +auth: + enabled: true + jwt: + issuerId: foxtrot-server + privateKey: ${PRIVATE_KEY} + sessionDuration: 15d + + provider: + type: OAUTH_IDMAN + idmanEndpoint: ${IDMAN_ENDPOINT} + clientId: FOXTROT + clientSecret: 8f4bcf45-2909-42e0-9bba-f46351bb0e6d + serverEndpoint: "http://localhost:17000" diff --git a/config/docker.yml b/config/docker.yml index 007e0e2ad..5457f9e89 100755 --- a/config/docker.yml +++ b/config/docker.yml @@ -6,6 +6,8 @@ server: - type: http port: 17001 +swaggerScheme: ${SWAGGER_SCHEME} + elasticsearch: hosts: - ${ELASTICSEARCH_HOST} @@ -25,7 +27,7 @@ cluster: discovery: type: ${DISCOVERY_TYPE} disableMulticast: true - members: ["localhost:5701"] + members: ["${HOSTNAME}:5701"] cacheConfig: maxIdleSeconds: 15 @@ -67,3 +69,8 @@ consoleHistoryConfig: interval: 86400 initialDelay: 10 +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + diff --git a/config/local.yml b/config/local.yml index 4c3c723d8..46df5b1a1 100755 --- a/config/local.yml +++ b/config/local.yml @@ -21,7 +21,7 @@ hbase: hbaseZookeeperQuorum: localhost:2181 hbaseZookeeperClientPort: 2181 seggregatedTablePrefix: foxtrot- - hbaseZookeeperZnodeParent: /hbase-unsecure + hbaseZookeeperZnodeParent: /hbase-test cluster: name: foxtrot @@ -50,6 +50,7 @@ elasticsearchTuningConfig: aggregationSize: 10000 scrollSize: 100 scrollTimeInSeconds: 120 + documentsLimitAllowed: 10000 esIndexOptimizationConfig: active: true @@ -64,4 +65,30 @@ consoleHistoryConfig: deletionconfig: active: false interval: 86400 - initialdelay: 60 \ No newline at end of file + initialdelay: 60 + + +clusterRerouteConfig: + active: false + interval: 86400 + initialDelay: 10 + thresholdShardCountPercentage: 10 + +sessionCleanupConfig: + active: true + interval: 86400 + initialDelay: 10 + +auth: + enabled: false + jwt: + issuerId: foxtrot-server + privateKey: ${PRIVATE_KEY} + sessionDuration: 15d + + provider: + type: OAUTH_GOOGLE + clientId: ${GOOGLE_CLIENT_ID} + clientSecret: ${GOOGLE_CLIENT_SECRET} + server: ${GOOGLE_CALLBACK_HOST_PORT} + secureEndpoint: false \ No newline at end of file diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml new file mode 100644 index 000000000..3073731d8 --- /dev/null +++ b/docker-compose-auth.yml @@ -0,0 +1,66 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8 + container_name: elasticsearch + hostname: elasticsearch + environment: + - cluster.name=elasticsearch + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + command: elasticsearch -Ehttp.cors.enabled=true + volumes: + - /elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + + hbase: + image: hyness/hbase-rest-standalone + hostname: hbase + ports: + - "2181:2181" + - "8080:8080" + - "16000:16000" + - "16010:16010" + - "16020:16020" + - "16030:16030" + volumes: + - /data/hbase + - /data/zookeeper + + foxtrot-server: + depends_on: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + - "5701:5701" + volumes: + - /var/log/foxtrot-server + environment: + - HOST=localhost + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_CLUSTER_NAME=elasticsearch + - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot + - ELASTICSEARCH_PORT="9200" + - ELASTICSEARCH_PROTOCOL=http + - HBASE_TABLE_NAME=foxtrot + - HBASE_ZOOKEEPER_QUORUM=hbase + - HBASE_ZOOKEEPER_CLIENT_PORT=2181 + - HAZELCAST_CLUSTER_NAME=foxtrot + - AUTH_ENABLED=true + - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - GOOGLE_SECURE_ENDPOINT=false + - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" diff --git a/docker-compose-gauth.yml b/docker-compose-gauth.yml new file mode 100644 index 000000000..dd9d1b6d0 --- /dev/null +++ b/docker-compose-gauth.yml @@ -0,0 +1,59 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8 + container_name: elasticsearch + hostname: elasticsearch + environment: + - cluster.name=elasticsearch + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + command: elasticsearch -Ehttp.cors.enabled=true + volumes: + - /elasticsearch/data + + hbase: + image: hyness/hbase-rest-standalone + hostname: hbase + volumes: + - /data/hbase + - /data/zookeeper + + foxtrot-server: + depends_on: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + - "5701:5701" + volumes: + - /var/log/foxtrot-server + environment: + - HOST=localhost + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9300 + - ELASTICSEARCH_CLUSTER_NAME=elasticsearch + - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot + - ELASTICSEARCH_PORT="9200" + - ELASTICSEARCH_PROTOCOL=http + - HBASE_TABLE_NAME=foxtrot + - HBASE_ZOOKEEPER_QUORUM=hbase + - HBASE_ZOOKEEPER_CLIENT_PORT=2181 + - HAZELCAST_CLUSTER_NAME=foxtrot + - GOOGLE_CALLBACK_HOST_PORT=localhost:17000 + - GOOGLE_CLIENT_ID=${GOOGLE_CLIENT_ID} + - GOOGLE_CLIENT_SECRET=${GOOGLE_CLIENT_SECRET} + - GOOGLE_SECURE_ENDPOINT=false + - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + - STARTUP_DELAY=15 + - CONFIG_PATH=/config/docker-gauth.yml + diff --git a/docker-compose-idman.yml b/docker-compose-idman.yml new file mode 100644 index 000000000..443d2e6d2 --- /dev/null +++ b/docker-compose-idman.yml @@ -0,0 +1,57 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.8 + container_name: elasticsearch + hostname: elasticsearch + environment: + - cluster.name=elasticsearch + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + command: elasticsearch -Ehttp.cors.enabled=true + volumes: + - /elasticsearch/data + + hbase: + image: hyness/hbase-rest-standalone + hostname: hbase + volumes: + - /data/hbase + - /data/zookeeper + + foxtrot-server: + depends_on: + - elasticsearch + - hbase + container_name: foxtrot_server + build: . + ports: + - "17000:17000" + - "17001:17001" + - "5701:5701" + volumes: + - /var/log/foxtrot-server + environment: + - HOST=localhost + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9300 + - ELASTICSEARCH_CLUSTER_NAME=elasticsearch + - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot + - ELASTICSEARCH_PORT="9200" + - ELASTICSEARCH_PROTOCOL=http + - HBASE_TABLE_NAME=foxtrot + - HBASE_ZOOKEEPER_QUORUM=hbase + - HBASE_ZOOKEEPER_CLIENT_PORT=2181 + - HAZELCAST_CLUSTER_NAME=foxtrot + - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - PRIVATE_KEY="6/Qh1eLvF7cGJEgH1wIMYErIpi0d8Kqt5wHKw/o5iFbhfziKiux03HfFihO2+f5FKpsjk1D7UmgdAWQsY966WkyAsyo22m4g+C+rNCwMUD/UM2qop7JZBDEkgWpwK8jpxOTpYUnaNsiBaBsadoZCQ+Ns/d8RrS7QPAW+1PETbdFfo+fPW0/Sm8RUmMKkc9jJ6bhuSpFH3q/bWoyPZnj7geYQQIkgmQv9jXaszw==" + - STARTUP_DELAY=15 + - CONFIG_PATH=/config/docker-idman.yml + - IDMAN_ENDPOINT=${IDMAN_ENDPOINT} + - IDMAN_SECRET=${IDMAN_SECRET} + diff --git a/docker-compose.yml b/docker-compose.yml index dfd2626ef..8de1255ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,20 +15,20 @@ services: command: elasticsearch -Ehttp.cors.enabled=true volumes: - /elasticsearch/data - ports: - - "9200:9200" - - "9300:9300" +# ports: +# - "9200:9200" +# - "9300:9300" hbase: image: hyness/hbase-rest-standalone hostname: hbase - ports: - - "2181:2181" - - "8080:8080" - - "16000:16000" - - "16010:16010" - - "16020:16020" - - "16030:16030" +# ports: +# - "2181:2181" +# - "8080:8080" +# - "16000:16000" +# - "16010:16010" +# - "16020:16020" +# - "16030:16030" volumes: - /data/hbase - /data/zookeeper @@ -46,8 +46,10 @@ services: volumes: - /var/log/foxtrot-server environment: + - INIT_SLEEP=15 - HOST=localhost - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9300 - ELASTICSEARCH_CLUSTER_NAME=elasticsearch - ELASTICSEARCH_TABLE_NAME_PREFIX=foxtrot - ELASTICSEARCH_PORT="9200" @@ -57,3 +59,5 @@ services: - HBASE_ZOOKEEPER_CLIENT_PORT=2181 - HAZELCAST_CLUSTER_NAME=foxtrot - DISCOVERY_TYPE=foxtrot_simple + - SWAGGER_SCHEME="http" + - STARTUP_DELAY=15 diff --git a/foxtrot-common/pom.xml b/foxtrot-common/pom.xml index 7221bbe3f..180266c54 100644 --- a/foxtrot-common/pom.xml +++ b/foxtrot-common/pom.xml @@ -5,7 +5,7 @@ com.flipkart.foxtrot foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java index f497b875f..2ced43036 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/Table.java @@ -32,9 +32,7 @@ * Representation for a table on foxtrot. */ @Data -@AllArgsConstructor @NoArgsConstructor -@Builder @JsonIgnoreProperties(ignoreUnknown = true) public class Table implements Serializable { @@ -46,8 +44,22 @@ public class Table implements Serializable { private String name; @Min(1) - @Max(180) private int ttl; private boolean seggregatedBackend = false; + + @Min(1) + @Max(256) + private int defaultRegions = 4; + + @Builder + public Table(String name, int ttl, boolean seggregatedBackend, int defaultRegions) { + this.name = name; + this.ttl = ttl; + this.seggregatedBackend = seggregatedBackend; + if (defaultRegions == 0) { + defaultRegions = 4; + } + this.defaultRegions = defaultRegions; + } } diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/hbase/HRegionData.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/hbase/HRegionData.java new file mode 100644 index 000000000..1150c7677 --- /dev/null +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/hbase/HRegionData.java @@ -0,0 +1,18 @@ +package com.flipkart.foxtrot.common.hbase; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class HRegionData { + String regionName; + String startKey; + String endKey; + byte[] encodedNameAsBytes; + long regionSize; +} diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java index f2e58ea27..cdce2e9d2 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiQueryRequest.java @@ -15,14 +15,11 @@ * limitations under the License. */ -import com.collections.CollectionUtils; import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionRequestVisitor; import com.flipkart.foxtrot.common.Opcodes; -import lombok.Data; -import org.junit.Assert; - import java.util.Map; +import lombok.Data; /*** Created by nitish.goyal on 22/08/18 @@ -38,7 +35,6 @@ public MultiQueryRequest() { public MultiQueryRequest(Map requests) { super(Opcodes.MULTI_QUERY); - Assert.assertTrue(CollectionUtils.isNotEmpty(requests)); this.requests = requests; } diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiTimeQueryRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiTimeQueryRequest.java index c80000108..902931985 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiTimeQueryRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/query/MultiTimeQueryRequest.java @@ -2,6 +2,7 @@ import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionRequestVisitor; +import com.flipkart.foxtrot.common.Opcodes; import io.dropwizard.util.Duration; import lombok.AllArgsConstructor; import lombok.Getter; @@ -26,6 +27,11 @@ public class MultiTimeQueryRequest extends ActionRequest { @NotNull private ActionRequest actionRequest; + + public MultiTimeQueryRequest() { + super(Opcodes.MULTI_TIME_QUERY); + } + public T accept(ActionRequestVisitor visitor) { return visitor.visit(this); } diff --git a/foxtrot-core/pom.xml b/foxtrot-core/pom.xml index b96b1c913..b8df1a363 100644 --- a/foxtrot-core/pom.xml +++ b/foxtrot-core/pom.xml @@ -5,7 +5,7 @@ com.flipkart.foxtrot foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 @@ -170,7 +170,7 @@ com.hazelcast hazelcast - 3.11.7 + ${hazelcast.version} @@ -182,7 +182,7 @@ com.hazelcast hazelcast - 3.11.7 + ${hazelcast.version} test-jar test diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java new file mode 100644 index 000000000..ed2723a95 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/FoxtrotRole.java @@ -0,0 +1,28 @@ +package com.flipkart.foxtrot.core.auth; + +import lombok.Getter; + +/** + * + */ +@Getter +public enum FoxtrotRole { + INGEST(Value.INGEST), + CONSOLE(Value.CONSOLE), + QUERY(Value.QUERY), + SYSADMIN(Value.SYSADMIN), + ; + + FoxtrotRole(String value) { + this.value = value; + } + + public static class Value { + public static final String INGEST = "INGEST"; + public static final String CONSOLE = "CONSOLE"; + public static final String QUERY = "QUERY"; + public static final String SYSADMIN = "SYSADMIN"; + } + + private final String value; +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java new file mode 100644 index 000000000..5a946a953 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/User.java @@ -0,0 +1,24 @@ +package com.flipkart.foxtrot.core.auth; + +import lombok.Value; + +import java.util.Collections; +import java.util.Date; +import java.util.EnumSet; +import java.util.Set; + +/** + * + */ +@Value +public class User { + public static final User DEFAULT + = new User("__DEFAULT__", EnumSet.allOf(FoxtrotRole.class), Collections.emptySet(), true, new Date(), new Date()); + + String id; + Set roles; + Set tables; + boolean systemUser; + Date created; + Date updated; +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java new file mode 100644 index 000000000..4457fedcb --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/auth/UserType.java @@ -0,0 +1,9 @@ +package com.flipkart.foxtrot.core.auth; + +/** + * + */ +public enum UserType { + HUMAN, + SYSTEM +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java index efa5d0dae..8733cb81e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCache.java @@ -20,7 +20,8 @@ import com.flipkart.foxtrot.common.ActionResponse; import com.flipkart.foxtrot.core.cache.Cache; import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; -import com.hazelcast.core.IMap; +import com.flipkart.foxtrot.core.util.Constants; +import com.hazelcast.map.IMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,8 +52,8 @@ public ActionResponse put(String key, ActionResponse data) { try { final String serializedData = mapper.writeValueAsString(data); if(serializedData != null) { - // Only cache if size is less that 32 KB - if(serializedData.length() <= 32 * 1024) { + // Only cache if size is less that 256 KB + if(serializedData.length() <= Constants.CACHE_VALUE_SIZE_IN_KB) { distributedMap.put(key, mapper.writeValueAsString(data)); } else { String responsePart = serializedData.substring(0, 1024); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java index a6cedf8c9..da6ee8ba5 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/cache/impl/DistributedCacheFactory.java @@ -20,10 +20,7 @@ import com.flipkart.foxtrot.core.cache.CacheFactory; import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; -import com.hazelcast.config.EvictionPolicy; -import com.hazelcast.config.InMemoryFormat; -import com.hazelcast.config.MapConfig; -import com.hazelcast.config.MaxSizeConfig; +import com.hazelcast.config.*; import javax.inject.Inject; import javax.inject.Singleton; @@ -60,7 +57,10 @@ private MapConfig getDefaultMapConfig(CacheConfig cacheConfig) { MapConfig mapConfig = new MapConfig(CACHE_NAME_PREFIX + "*"); mapConfig.setInMemoryFormat(InMemoryFormat.BINARY); mapConfig.setBackupCount(0); - mapConfig.setEvictionPolicy(EvictionPolicy.LRU); + final EvictionConfig evictionConfig = mapConfig.getEvictionConfig(); + evictionConfig.setEvictionPolicy(EvictionPolicy.LRU); + evictionConfig.setSize(cacheConfig.getSize() == 0 ? DEFAULT_SIZE : cacheConfig.getSize()); + evictionConfig.setMaxSizePolicy(MaxSizePolicy.USED_HEAP_SIZE); if(cacheConfig.getMaxIdleSeconds() == 0) { mapConfig.setMaxIdleSeconds(DEFAULT_MAX_IDLE_SECONDS); @@ -74,14 +74,6 @@ private MapConfig getDefaultMapConfig(CacheConfig cacheConfig) { mapConfig.setTimeToLiveSeconds(cacheConfig.getTimeToLiveSeconds()); } - MaxSizeConfig maxSizeConfig = new MaxSizeConfig(); - maxSizeConfig.setMaxSizePolicy(MaxSizeConfig.MaxSizePolicy.USED_HEAP_PERCENTAGE); - if(cacheConfig.getSize() == 0) { - maxSizeConfig.setSize(DEFAULT_SIZE); - } else { - maxSizeConfig.setSize(cacheConfig.getSize()); - } - mapConfig.setMaxSizeConfig(maxSizeConfig); return mapConfig; } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java index fd5181fe7..30344c41d 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/common/Action.java @@ -144,7 +144,7 @@ private void validateBase(P parameter) { public abstract String getRequestCacheKey(); - public abstract org.elasticsearch.action.ActionRequest getRequestBuilder(P parameter); + public abstract org.elasticsearch.action.ActionRequest getRequestBuilder(P parameter, List extraFilters); public abstract ActionResponse getResponse(org.elasticsearch.action.ActionResponse response, P parameter); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java index b002ecb65..f5c440629 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/DataStore.java @@ -36,4 +36,6 @@ public interface DataStore { Document get(final Table table, final String id); List getAll(final Table table, final List ids); + + void updateTable(final Table table); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java index 3a6e4e5df..8720f61e8 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseDataStore.java @@ -218,10 +218,9 @@ public List getAll(final Table table, List ids) { missingIds.add(ids.get(index)); } } - if(!missingIds.isEmpty()) { - String allIds = String.join(",", ids); - logger.error("ID's missing in HBase - {}", allIds); - throw FoxtrotExceptions.createMissingDocumentsException(table, ids); + if (!missingIds.isEmpty()) { + logger.error("ID's missing in HBase - {}", missingIds); + throw FoxtrotExceptions.createMissingDocumentsException(table, missingIds); } return results; } catch (JsonProcessingException e) { @@ -242,4 +241,19 @@ public Put getPutForDocument(Document document) throws JsonProcessingException { .addColumn(COLUMN_FAMILY, DATE_FIELD_NAME, mapper.writeValueAsBytes(document.getDate())); } + @Override + @Timed + public void updateTable(final Table table) { + // Check for existence of HBase table during update to make sure HBase is ready for taking writes + try { + boolean isTableAvailable = tableWrapper.isTableAvailable(table); + if (isTableAvailable) { + tableWrapper.updateTable(table); + } + } + catch (IOException e) { + throw FoxtrotExceptions.createConnectionException(table, e); + } + } + } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java index 632feb331..291076e8b 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HBaseUtil.java @@ -17,6 +17,7 @@ package com.flipkart.foxtrot.core.datastore.impl.hbase; import com.google.common.base.Strings; +import lombok.experimental.UtilityClass; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -33,11 +34,32 @@ import java.io.File; import java.io.IOException; -public abstract class HBaseUtil { +@UtilityClass +public class HBaseUtil { private static final Logger logger = LoggerFactory.getLogger(HBaseUtil.class); - private HBaseUtil() {} + public static void createTable(final HbaseConfig hbaseConfig, final String tableName) throws IOException { + final TableName name = TableName.valueOf(tableName); + HTableDescriptor hTableDescriptor = new HTableDescriptor(name); + HColumnDescriptor columnDescriptor = new HColumnDescriptor("d"); + columnDescriptor.setCompressionType(Compression.Algorithm.GZ); + hTableDescriptor.addFamily(columnDescriptor); + Configuration configuration = HBaseUtil.create(hbaseConfig); + try(Connection connection = ConnectionFactory.createConnection(configuration)) { + try (Admin admin = connection.getAdmin()) { + if (admin.tableExists(TableName.valueOf(tableName))) { + logger.info("Table {} already exists. Nothing to do.", tableName); + return; + } + logger.info("Creating table: {}", tableName); + admin.createTable(hTableDescriptor); + } + catch (Exception e) { + logger.error("Could not create table: " + tableName, e); + } + } + } public static Configuration create(final HbaseConfig hbaseConfig) throws IOException { Configuration configuration = HBaseConfiguration.create(); @@ -89,35 +111,4 @@ public static boolean isValidFile(String fileName) { return fileName != null && !fileName.trim() .isEmpty() && new File(fileName).exists(); } - - public static void createTable(final HbaseConfig hbaseConfig, final String tableName) throws IOException { - HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); - HColumnDescriptor columnDescriptor = new HColumnDescriptor("d"); - columnDescriptor.setCompressionType(Compression.Algorithm.GZ); - hTableDescriptor.addFamily(columnDescriptor); - Configuration configuration = HBaseUtil.create(hbaseConfig); - Connection connection = ConnectionFactory.createConnection(configuration); - Admin hBaseAdmin = null; - try { - hBaseAdmin = connection.getAdmin(); - if(!hBaseAdmin.tableExists(TableName.valueOf(tableName))) { - hBaseAdmin.createTable(hTableDescriptor); - } - } catch (Exception e) { - logger.error("Could not create table: " + tableName, e); - } finally { - try { - if(hBaseAdmin != null) { - hBaseAdmin.close(); - } - } catch (Exception e) { - logger.error("Error closing hbase admin", e); - } - try { - connection.close(); - } catch (Exception e) { - logger.error("Error closing hbase connection", e); - } - } - } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseRegions.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseRegions.java new file mode 100644 index 000000000..a73c86c35 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseRegions.java @@ -0,0 +1,142 @@ +package com.flipkart.foxtrot.core.datastore.impl.hbase; + +import com.flipkart.foxtrot.common.hbase.HRegionData; +import com.flipkart.foxtrot.common.util.CollectionUtils; +import com.flipkart.foxtrot.core.exception.HbaseRegionExtractionException; +import com.flipkart.foxtrot.core.exception.HbaseRegionMergeException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.util.RegionSizeCalculator; + +import java.util.*; + +@Slf4j +public class HbaseRegions { + + private static final long BYTES_IN_GB = 1073741824; + private Connection connection; + private Admin hBaseAdmin; + + public HbaseRegions(HbaseConfig hbaseConfig) { + try { + Configuration configuration = HBaseUtil.create(hbaseConfig); + connection = ConnectionFactory.createConnection(configuration); + this.hBaseAdmin = connection.getAdmin(); + } catch (Exception e) { + log.info("Unable to get Hbase Admin!!!", e); + } + } + + public List> getMergeableRegions(TableName tablename, double threshSizeInGB) { + long threshSize = (long)(threshSizeInGB * BYTES_IN_GB); + Map hash = getRegionsMap(tablename); + if (hash.isEmpty()) { + return Collections.emptyList(); + } + try { + HRegionData currentRegion = hash.get(""); + HRegionData nextRegion; + + List> mergeRegions = new ArrayList<>(); + + while (!StringUtils.isEmpty(currentRegion.getEndKey())) { + nextRegion = hash.get(currentRegion.getEndKey()); + if (currentRegion.getRegionSize() + nextRegion.getRegionSize() <= threshSize) { + mergeRegions.add(Arrays.asList(currentRegion, nextRegion)); + if (StringUtils.isEmpty(nextRegion.getEndKey())) { + break; + } + currentRegion = hash.get(nextRegion.getEndKey()); + } else { + currentRegion = nextRegion; + } + } + return mergeRegions; + } catch (Exception e) { + throw new HbaseRegionMergeException("Unable to retrieve mergeable regions", e); + } + } + + public void mergeRegions(TableName tablename, double threshSizeInGB, int numberOfMerges) { + long threshSize = (long)(threshSizeInGB * BYTES_IN_GB); + if (numberOfMerges == -1) { + numberOfMerges = Integer.MAX_VALUE; + } + Map hash = getRegionsMap(tablename); + if (hash.isEmpty()) { + log.info("No regions to merge!!"); + return; + } + try { + HRegionData currentRegion = hash.get(""); + HRegionData nextRegion; + int count = 0; + + while (!StringUtils.isEmpty(currentRegion.getEndKey()) && count < numberOfMerges) { + nextRegion = hash.get(currentRegion.getEndKey()); + if (currentRegion.getRegionSize() + nextRegion.getRegionSize() <= threshSize) { + log.info(String.format("Starting to Merge regions : %s startKey : %s endKey : %s and %s startKey : %s endKey : %s", + currentRegion.getRegionName(), + currentRegion.getStartKey(), + currentRegion.getEndKey(), + nextRegion.getRegionName(), + nextRegion.getStartKey(), + nextRegion.getEndKey())); + hBaseAdmin.mergeRegions(currentRegion.getEncodedNameAsBytes(), nextRegion.getEncodedNameAsBytes(), false); + log.info("Merged!!!"); + count++; + if (StringUtils.isEmpty(nextRegion.getEndKey())) { + break; + } + currentRegion = hash.get(nextRegion.getEndKey()); + } else { + currentRegion = nextRegion; + } + } + } catch (Exception e) { + throw new HbaseRegionMergeException("Error merging regions!!", e); + } + } + + private Map getRegionsMap(TableName tablename) { + try { + List regions = getRegions(tablename); + RegionLocator regionLocator = connection.getRegionLocator(tablename); + RegionSizeCalculator regionSizeCalculator = new RegionSizeCalculator(regionLocator, hBaseAdmin); + Map regionsMap = new HashMap<>(); + + if (CollectionUtils.isNullOrEmpty(regions)) { + return Collections.emptyMap(); + } + for (HRegionInfo region : regions) { + regionsMap.put(new String(region.getStartKey()), + HRegionData.builder() + .regionName(region.getRegionNameAsString()) + .startKey(new String(region.getStartKey())) + .endKey(new String(region.getEndKey())) + .encodedNameAsBytes(region.getEncodedNameAsBytes()) + .regionSize(regionSizeCalculator.getRegionSize(region.getRegionName())) + .build() + ); + } + return regionsMap; + } catch (Exception e) { + throw new HbaseRegionExtractionException("Unable to retrieve region sizes", e); + } + } + + private List getRegions(TableName tablename) { + try { + return hBaseAdmin.getTableRegions(tablename); + } catch (Exception e) { + throw new HbaseRegionExtractionException("Unable to retrieve regions", e); + } + } +} \ No newline at end of file diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java index 3a5e05d44..519c429af 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/datastore/impl/hbase/HbaseTableConnection.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.util.RegionSplitter; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +36,7 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.io.IOException; +import java.util.concurrent.TimeUnit; /** * User: Santanu Sinha (santanu.sinha@flipkart.com) @@ -77,13 +79,16 @@ public synchronized boolean isTableAvailable(final Table table) throws IOExcepti } public synchronized void createTable(final Table table) throws IOException { + HTableDescriptor hTableDescriptor = constructHTableDescriptor(table); + byte[][] splits = new RegionSplitter.HexStringSplit().split(table.getDefaultRegions()); + hBaseAdmin.createTable(hTableDescriptor, splits); + } + + public synchronized void updateTable(final Table table) throws IOException { String tableName = TableUtil.getTableName(hbaseConfig, table); - HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); - HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(DEFAULT_FAMILY_NAME); - hColumnDescriptor.setCompressionType(Compression.Algorithm.GZ); - hTableDescriptor.addFamily(hColumnDescriptor); - hBaseAdmin.createTable(hTableDescriptor); + HTableDescriptor hTableDescriptor = constructHTableDescriptor(table); + hBaseAdmin.modifyTable(TableName.valueOf(tableName), hTableDescriptor); } public String getHBaseTableName(final Table table) { @@ -93,6 +98,7 @@ public String getHBaseTableName(final Table table) { @Override public void start() throws Exception { logger.info("Starting HBase Connection"); + Configuration configuration = HBaseUtil.create(hbaseConfig); connection = ConnectionFactory.createConnection(configuration); this.hBaseAdmin = connection.getAdmin(); logger.info("Started HBase Connection"); @@ -107,4 +113,15 @@ public void stop() throws Exception { public HbaseConfig getHbaseConfig() { return hbaseConfig; } + + private HTableDescriptor constructHTableDescriptor(final Table table) { + String tableName = TableUtil.getTableName(hbaseConfig, table); + + HTableDescriptor hTableDescriptor = new HTableDescriptor(TableName.valueOf(tableName)); + HColumnDescriptor hColumnDescriptor = new HColumnDescriptor(DEFAULT_FAMILY_NAME); + hColumnDescriptor.setCompressionType(Compression.Algorithm.GZ); + hColumnDescriptor.setTimeToLive(Math.toIntExact(TimeUnit.DAYS.toSeconds(table.getTtl()))); + hTableDescriptor.addFamily(hColumnDescriptor); + return hTableDescriptor; + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ConsoleQueryBlockedException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ConsoleQueryBlockedException.java new file mode 100644 index 000000000..d65fcb752 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ConsoleQueryBlockedException.java @@ -0,0 +1,30 @@ +package com.flipkart.foxtrot.core.exception; + +import com.flipkart.foxtrot.common.ActionRequest; +import com.google.common.collect.Maps; + +import java.util.Map; + +/*** + Created by nitish.goyal on 10/09/19 + ***/ +public class ConsoleQueryBlockedException extends FoxtrotException { + + private static final long serialVersionUID = -8591567152701424684L; + + private final ActionRequest actionRequest; + + public ConsoleQueryBlockedException(ActionRequest actionRequest) { + super(ErrorCode.CONSOLE_QUERY_BLOCKED, + "Console Query blocked due to high load. Kindly run after sometime"); + this.actionRequest = actionRequest; + } + + @Override + public Map toMap() { + Map map = Maps.newHashMap(); + map.put("request", this.actionRequest); + return map; + } + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ErrorCode.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ErrorCode.java index 217679819..1e5738d5b 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ErrorCode.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/ErrorCode.java @@ -35,5 +35,15 @@ public enum ErrorCode { FQL_PERSISTENCE_EXCEPTION, FQL_PARSE_ERROR, - PORT_EXTRACTION_ERROR + PORT_EXTRACTION_ERROR, + PERMISSION_CREATION_FAILURE, + ENDPOINT_NOT_FOUND_ERROR, + USER_NOT_FOUND, + USER_PERMISSION_ADDITION_FAILURE, + AUTH_TOKEN_EXCEPTION, + + CONSOLE_QUERY_BLOCKED, + + HBASE_REGIONS_EXTRACTION_FAILURE, + HBASE_REGIONS_MERGE_FAILURE } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/FoxtrotException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/FoxtrotException.java index a4554c976..50418d3cc 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/FoxtrotException.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/FoxtrotException.java @@ -1,5 +1,6 @@ package com.flipkart.foxtrot.core.exception; +import com.google.common.collect.Maps; import lombok.Getter; import java.util.Map; @@ -31,6 +32,10 @@ protected FoxtrotException(ErrorCode code, String message, Throwable cause) { this.code = code; } - public abstract Map toMap(); + public Map toMap() { + Map map = Maps.newHashMap(); + map.put("errorCode", code); + return map; + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/HbaseRegionExtractionException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/HbaseRegionExtractionException.java new file mode 100644 index 000000000..b3ddd3875 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/HbaseRegionExtractionException.java @@ -0,0 +1,19 @@ +package com.flipkart.foxtrot.core.exception; + +import java.util.Map; + +public class HbaseRegionExtractionException extends FoxtrotException { + private final String message; + + public HbaseRegionExtractionException(String message, Throwable cause) { + super(ErrorCode.HBASE_REGIONS_EXTRACTION_FAILURE, message, cause); + this.message = message; + } + + @Override + public Map toMap() { + Map map = super.toMap(); + map.put("message", message); + return map; + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/HbaseRegionMergeException.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/HbaseRegionMergeException.java new file mode 100644 index 000000000..07c8a8d3b --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/exception/HbaseRegionMergeException.java @@ -0,0 +1,21 @@ +package com.flipkart.foxtrot.core.exception; + +import com.google.common.collect.Maps; + +import java.util.Map; + +public class HbaseRegionMergeException extends FoxtrotException { + private final String message; + + public HbaseRegionMergeException(String message, Throwable cause) { + super(ErrorCode.HBASE_REGIONS_MERGE_FAILURE, message, cause); + this.message = message; + } + + @Override + public Map toMap() { + Map map = Maps.newHashMap(); + map.put("message", message); + return map; + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java index d493f009d..ec39ec5b8 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/EventPublisherActionExecutionObserver.java @@ -17,6 +17,9 @@ public EventPublisherActionExecutionObserver(InternalEventBus eventBus) { @Override public void postExecution(ActionEvaluationResponse response) { + if (null == response){ + return; + } if(null != response.getException()) { eventBus.publish(new QueryProcessingError(response.getRequest(), response.getException())); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java index e4c66617b..989f29c20 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/QueryExecutor.java @@ -63,7 +63,7 @@ public ActionValidationResponse validate(T request) { public ActionResponse execute(T request) { Stopwatch stopwatch = Stopwatch.createStarted(); - Action action = null; + Action action = null; ActionEvaluationResponse evaluationResponse = null; try { action = resolve(request); @@ -108,8 +108,8 @@ public AsyncDataToken executeAsync(T request) { return dataToken; } - public Action resolve(T request) { - Action action; + public Action resolve(T request) { + Action action; action = analyticsLoader.getAction(request); if(null == action) { throw FoxtrotExceptions.createUnresolvableActionException(request); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java index ee6c6816c..614a9f92e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/CountAction.java @@ -13,7 +13,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils.QUERY_SIZE; @@ -97,7 +98,7 @@ public void validateImpl(CountRequest parameter) { @Override public ActionResponse execute(CountRequest parameter) { - SearchRequest request = getRequestBuilder(parameter); + SearchRequest request = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() @@ -112,14 +113,14 @@ public ActionResponse execute(CountRequest parameter) { } @Override - public SearchRequest getRequestBuilder(CountRequest parameter) { + public SearchRequest getRequestBuilder(CountRequest parameter, List extraFilters) { if (parameter.isDistinct()) { try { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(Utils.buildCardinalityAggregation(parameter.getField(), parameter.accept(new CountPrecisionThresholdVisitorAdapter( elasticsearchTuningConfig.getPrecisionThreshold()))))) @@ -135,7 +136,7 @@ public SearchRequest getRequestBuilder(CountRequest parameter) { .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters()))); + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters))); } catch (Exception e) { throw FoxtrotExceptions.queryCreationException(parameter, e); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java index 06028f2eb..d54885497 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/DistinctAction.java @@ -12,7 +12,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Sets; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -106,7 +107,7 @@ public void validateImpl(DistinctRequest parameter) { public ActionResponse execute(DistinctRequest request) { SearchRequest query; try { - query = getRequestBuilder(request); + query = getRequestBuilder(request, Collections.emptyList()); } catch (Exception e) { throw FoxtrotExceptions.queryCreationException(request, e); @@ -125,12 +126,12 @@ public ActionResponse execute(DistinctRequest request) { } @Override - public SearchRequest getRequestBuilder(DistinctRequest request) { + public SearchRequest getRequestBuilder(DistinctRequest request, List extraFilters) { try { return new SearchRequest(ElasticsearchUtils.getIndices(request.getTable(), request)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() - .query(new ElasticSearchQueryGenerator().genFilter(request.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(request, extraFilters)) .size(QUERY_SIZE) .aggregation(Utils.buildTermsAggregation( request.getNesting(), Sets.newHashSet(), elasticsearchTuningConfig.getAggregationSize())) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java index cb49f25d3..b23915cd6 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/FilterAction.java @@ -27,7 +27,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -125,14 +125,14 @@ public void validateImpl(Query parameter) { @Override public ActionResponse execute(Query parameter) { if (parameter.isScrollRequest()) { - return executeScrollRequest(parameter); + return executeScrollRequest(parameter, Collections.emptyList()); } return executeRequest(parameter); } @Override - public SearchRequest getRequestBuilder(Query parameter) { - SearchRequest searchRequest = getSearchRequest(parameter); + public SearchRequest getRequestBuilder(Query parameter, List extraFilters) { + SearchRequest searchRequest = getSearchRequest(parameter, extraFilters); //Adding from here since from isn't supported in scroll request searchRequest.source().from(parameter.getFrom()); return searchRequest; @@ -161,13 +161,13 @@ public ActionResponse getResponse(org.elasticsearch.action.ActionResponse respon } - private SearchRequest getScrollRequestBuilder(Query parameter) { - SearchRequest searchRequest = getSearchRequest(parameter); + private SearchRequest getScrollRequestBuilder(Query parameter, List extraFilters) { + SearchRequest searchRequest = getSearchRequest(parameter, extraFilters); searchRequest.scroll(TimeValue.timeValueSeconds(elasticsearchTuningConfig.getScrollTimeInSeconds())); return searchRequest; } - private SearchRequest getSearchRequest(Query parameter) { + private SearchRequest getSearchRequest(Query parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .types(ElasticsearchUtils.DOCUMENT_TYPE_NAME) @@ -175,13 +175,13 @@ private SearchRequest getSearchRequest(Query parameter) { .source(new SearchSourceBuilder() .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) .size(parameter.getLimit()) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .sort(Utils.storedFieldName(parameter.getSort().getField()), ResultSort.Order.desc == parameter.getSort().getOrder() ? SortOrder.DESC : SortOrder.ASC)); } - private ActionResponse executeScrollRequest(Query parameter) { + private ActionResponse executeScrollRequest(Query parameter, List extraFilters) { List ids = new ArrayList<>(); String scrollId; long totalHits = 0; @@ -199,7 +199,7 @@ private ActionResponse executeScrollRequest(Query parameter) { } } else { - SearchRequest searchRequest = getScrollRequestBuilder(parameter); + SearchRequest searchRequest = getScrollRequestBuilder(parameter, extraFilters); SearchResponse searchResponse = getConnection() .getClient() .search(searchRequest, RequestOptions.DEFAULT); @@ -237,7 +237,7 @@ private QueryResponse getResponse(Query parameter, long totalHits, List } private ActionResponse executeRequest(Query parameter) { - SearchRequest search = getRequestBuilder(parameter); + SearchRequest search = getRequestBuilder(parameter, Collections.emptyList()); try { logger.info("Search: {}", search); SearchResponse response = getConnection() diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index ab9b54466..3eb9abf8e 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -42,8 +42,8 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchQueryStore; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.flipkart.foxtrot.core.table.TableMetadataManager; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.SneakyThrows; @@ -154,7 +154,7 @@ public void validateImpl(GroupRequest parameter) { @Override public ActionResponse execute(GroupRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -167,13 +167,13 @@ public ActionResponse execute(GroupRequest parameter) { } @Override - public SearchRequest getRequestBuilder(GroupRequest parameter) { + public SearchRequest getRequestBuilder(GroupRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java index 3e4a47e09..51d464f11 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/HistogramAction.java @@ -28,7 +28,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import io.dropwizard.util.Duration; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -121,7 +121,7 @@ public void validateImpl(HistogramRequest parameter) { @Override public ActionResponse execute(HistogramRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -134,13 +134,13 @@ public ActionResponse execute(HistogramRequest parameter) { } @Override - public SearchRequest getRequestBuilder(HistogramRequest parameter) { + public SearchRequest getRequestBuilder(HistogramRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java index be28eb59a..9fe768e44 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryAction.java @@ -3,6 +3,7 @@ import com.collections.CollectionUtils; import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.query.MultiQueryRequest; import com.flipkart.foxtrot.common.query.MultiQueryResponse; import com.flipkart.foxtrot.core.common.Action; @@ -11,27 +12,30 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.elasticsearch.action.search.MultiSearchRequest; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; import org.glassfish.hk2.api.MultiException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /*** Created by nitish.goyal on 22/08/18 ***/ @AnalyticsProvider(opcode = "multi_query", request = MultiQueryRequest.class, response = MultiQueryResponse.class, cacheable = true) +@Slf4j public class MultiQueryAction extends Action { - private static final Logger LOGGER = LoggerFactory.getLogger(MultiQueryAction.class); private AnalyticsLoader analyticsLoader; private Map requestActionMap = Maps.newHashMap(); @@ -57,8 +61,17 @@ public String getMetricKey() { @Override public String getRequestCacheKey() { - createActions(getParameter()); - return processForSubQueries(getParameter(), (action, request) -> action.getRequestCacheKey()); + final MultiQueryRequest parameter = getParameter(); + createActions(parameter); + long filterHashKey = 0L; + if (null != parameter.getFilters()) { + for (Filter filter : parameter.getFilters()) { + filterHashKey += 31 * filter.hashCode(); + } + } + return String.format("multquery-%d-%s", + filterHashKey, + processForSubQueries(parameter, (action, request) -> action.getRequestCacheKey())); } @Override @@ -81,12 +94,24 @@ public void validateImpl(MultiQueryRequest parameter) { @Override public ActionResponse execute(MultiQueryRequest parameter) { - MultiSearchRequest multiSearchRequestBuilder = getRequestBuilder(parameter); + if(Utils.hasTemporalFilters(parameter.getFilters())) { + val offendingRequests = parameter.getRequests().entrySet().stream() + .filter(entry -> Utils.hasTemporalFilters(entry.getValue().getFilters())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(offendingRequests)) { + throw FoxtrotExceptions.createMalformedQueryException( + parameter, + Collections.singletonList( + "Temporal filters passed in multi query as well as children: " + offendingRequests)); + } + } + MultiSearchRequest multiSearchRequestBuilder = getRequestBuilder(parameter, Collections.emptyList()); try { - LOGGER.info("Search: {}", multiSearchRequestBuilder); + log.info("Search: {}", multiSearchRequestBuilder); MultiSearchResponse multiSearchResponse = getConnection() .getClient() - .multiSearch(multiSearchRequestBuilder); + .multiSearch(multiSearchRequestBuilder, RequestOptions.DEFAULT); return getResponse(multiSearchResponse, parameter); } catch (IOException e) { throw FoxtrotExceptions.createQueryExecutionException(parameter, e); @@ -94,10 +119,17 @@ public ActionResponse execute(MultiQueryRequest parameter) { } @Override - public MultiSearchRequest getRequestBuilder(MultiQueryRequest parameter) { + public MultiSearchRequest getRequestBuilder(MultiQueryRequest parameter, List extraFilters) { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); - + val filterBuilder = ImmutableList.builder(); + if(null != parameter.getFilters()) { + filterBuilder.addAll(parameter.getFilters()); + } + if(null != extraFilters) { + filterBuilder.addAll(extraFilters); + } + val filters = filterBuilder.build(); for(Map.Entry entry : parameter.getRequests() .entrySet()) { @@ -106,7 +138,7 @@ public MultiSearchRequest getRequestBuilder(MultiQueryRequest parameter) { if(null == action) { throw FoxtrotExceptions.queryCreationException(request, null); } - org.elasticsearch.action.ActionRequest requestBuilder = action.getRequestBuilder(request); + org.elasticsearch.action.ActionRequest requestBuilder = action.getRequestBuilder(request, filters); if(requestBuilder instanceof SearchRequest) { multiSearchRequest.add((SearchRequest) requestBuilder); } @@ -132,7 +164,7 @@ public ActionResponse getResponse(org.elasticsearch.action.ActionResponse multiS try { action = analyticsLoader.getAction(request); } catch (Exception e) { - LOGGER.error("Error occurred while executing multiQuery request : {}", e); + log.error("Error occurred while executing multiQuery request : {}", e); } if(null == action) { throw FoxtrotExceptions.queryCreationException(request, null); @@ -145,6 +177,19 @@ public ActionResponse getResponse(org.elasticsearch.action.ActionResponse multiS } private void createActions(final MultiQueryRequest multiQueryRequest) { + if(Utils.hasTemporalFilters(multiQueryRequest.getFilters())) { + val offendingRequests = multiQueryRequest.getRequests().entrySet().stream() + .filter(entry -> Utils.hasTemporalFilters(entry.getValue().getFilters())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + if(CollectionUtils.isNotEmpty(offendingRequests)) { + throw FoxtrotExceptions.createMalformedQueryException( + multiQueryRequest, + Collections.singletonList( + "Temporal filters passed in multi query as well as children: " + offendingRequests)); + } + } + for(Map.Entry entry : multiQueryRequest.getRequests().entrySet()) { ActionRequest request = entry.getValue(); Action action; @@ -159,12 +204,17 @@ private void createActions(final MultiQueryRequest multiQueryRequest) { throw FoxtrotExceptions.createMalformedQueryException(multiQueryRequest, Collections.singletonList( "No action found for the sub request : " + request.toString())); } + } } private String processForSubQueries(MultiQueryRequest multiQueryRequest, ActionInterface actionInterface) { List results = Lists.newArrayList(); for(Map.Entry entry : multiQueryRequest.getRequests().entrySet()) { + if(null == entry.getValue()) { + log.warn("Empty response for query: {}", entry.getKey()); + continue; + } String result = actionInterface.invoke(requestActionMap.get(entry.getValue()), entry.getValue()); if(!Strings.isNullOrEmpty(result)) { results.add(result); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java index 0aa91e1f3..8f13de83c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/MultiTimeQueryAction.java @@ -115,8 +115,8 @@ public ActionResponse execute(MultiTimeQueryRequest parameter) { } @Override - public org.elasticsearch.action.ActionRequest getRequestBuilder(MultiTimeQueryRequest parameter) { - return action.getRequestBuilder(multiQueryRequest); + public org.elasticsearch.action.ActionRequest getRequestBuilder(MultiTimeQueryRequest parameter, List extraFilters) { + return action.getRequestBuilder(multiQueryRequest, extraFilters); } @Override diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java index 05fb3e659..f9bfafd1c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsAction.java @@ -11,7 +11,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Lists; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -107,7 +107,7 @@ public void validateImpl(StatsRequest parameter) { @Override public ActionResponse execute(StatsRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -120,11 +120,11 @@ public ActionResponse execute(StatsRequest parameter) { } @Override - public SearchRequest getRequestBuilder(StatsRequest parameter) { + public SearchRequest getRequestBuilder(StatsRequest parameter, List extraFilters) { final SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() .size(0) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())); + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)); AbstractAggregationBuilder percentiles = null; final String field = getParameter().getField(); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java index 2e7e96cd6..9b3a85bf1 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/StatsTrendAction.java @@ -13,7 +13,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.dropwizard.util.Duration; @@ -119,7 +119,7 @@ public void validateImpl(StatsTrendRequest parameter) { @Override public ActionResponse execute(StatsTrendRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -132,13 +132,13 @@ public ActionResponse execute(StatsTrendRequest parameter) { } @Override - public SearchRequest getRequestBuilder(StatsTrendRequest parameter) { + public SearchRequest getRequestBuilder(StatsTrendRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(0) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter, parameter.getTable()))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java index 7ce3d2160..dc620b464 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/TrendAction.java @@ -31,7 +31,7 @@ import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsLoader; import com.flipkart.foxtrot.core.querystore.actions.spi.AnalyticsProvider; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.dropwizard.util.Duration; @@ -150,7 +150,7 @@ public void validateImpl(TrendRequest parameter) { @Override public ActionResponse execute(TrendRequest parameter) { - SearchRequest query = getRequestBuilder(parameter); + SearchRequest query = getRequestBuilder(parameter, Collections.emptyList()); try { SearchResponse response = getConnection() .getClient() @@ -163,13 +163,13 @@ public ActionResponse execute(TrendRequest parameter) { } @Override - public SearchRequest getRequestBuilder(TrendRequest parameter) { + public SearchRequest getRequestBuilder(TrendRequest parameter, List extraFilters) { return new SearchRequest(ElasticsearchUtils.getIndices(parameter.getTable(), parameter)) .indicesOptions(Utils.indicesOptions()) .source(new SearchSourceBuilder() .size(QUERY_SIZE) .timeout(new TimeValue(getGetQueryTimeout(), TimeUnit.MILLISECONDS)) - .query(new ElasticSearchQueryGenerator().genFilter(parameter.getFilters())) + .query(ElasticsearchQueryUtils.translateFilter(parameter, extraFilters)) .aggregation(buildAggregation(parameter))); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java index 15bcf98ae..78f745dc9 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java @@ -5,6 +5,7 @@ import com.flipkart.foxtrot.common.FieldType; import com.flipkart.foxtrot.common.Period; import com.flipkart.foxtrot.common.TableFieldMapping; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.query.ResultSort; import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.common.util.CollectionUtils; @@ -353,4 +354,10 @@ public static boolean isNumericField(TableMetadataManager tableMetadataManager, return null != fieldMetadata && NUMERIC_FIELD_TYPES.contains(fieldMetadata.getType()); } + public static boolean hasTemporalFilters(List filters) { + if(null == filters) { + return false; + } + return filters.stream().anyMatch(Filter::isFilterTemporal); + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/MetricRecorder.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/MetricRecorder.java index f5f87dadd..d06f0e87c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/MetricRecorder.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/MetricRecorder.java @@ -11,7 +11,7 @@ public class MetricRecorder implements ActionExecutionObserver { @Override public void postExecution(ActionEvaluationResponse response) { - if(null == response || null == response.getExecutedAction()) { + if(null == response || null == response.getExecutedAction()) { return; } final ActionRequest request = response.getRequest(); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java index b03ec13c5..850b715fa 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/handlers/SlowQueryReporter.java @@ -12,7 +12,7 @@ public class SlowQueryReporter implements ActionExecutionObserver { @Override public void postExecution(ActionEvaluationResponse response) { - if(null != response.getException() || response.isCached()) { + if(null == response || null != response.getException() || response.isCached()) { return; } if (response.getElapsedTime() > 1000) { diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java index 8fa57489a..1d24ef1d6 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsClusterDiscoveryConfig.java @@ -16,104 +16,50 @@ package com.flipkart.foxtrot.core.querystore.impl; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +import javax.validation.constraints.Min; /** - * @author phaneesh + * AWS EC2 based cluster configuration. + * See: https://github.com/hazelcast/hazelcast-aws#ecsfargate-configuration */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) public class AwsClusterDiscoveryConfig extends ClusterDiscoveryConfig { + @JsonProperty + private String serviceName; @JsonProperty private String accessKey; + @JsonProperty private String secretKey; + @JsonProperty - private String region = "us-east-1"; - @JsonProperty - private String securityGroupName; + private String iamRole; + @JsonProperty - private String tagKey; + private String region; + @JsonProperty - private String tagValue; + private String hostHeader; + @JsonProperty - private String hostHeader = "ec2.amazonaws.com"; + private String securityGroupName; + @JsonProperty - private String iamRole; + @Min(0) + private int opTimeoutSeconds; + @JsonProperty - private int connectionTimeoutSeconds = 5; + private boolean isExternalClient; public AwsClusterDiscoveryConfig() { super(ClusterDiscoveryType.FOXTROT_AWS); } - public String getAccessKey() { - return accessKey; - } - - public void setAccessKey(String accessKey) { - this.accessKey = accessKey; - } - - public String getSecretKey() { - return secretKey; - } - - public void setSecretKey(String secretKey) { - this.secretKey = secretKey; - } - - public String getRegion() { - return region; - } - - public void setRegion(String region) { - this.region = region; - } - - public String getSecurityGroupName() { - return securityGroupName; - } - - public void setSecurityGroupName(String securityGroupName) { - this.securityGroupName = securityGroupName; - } - - public String getTagKey() { - return tagKey; - } - - public void setTagKey(String tagKey) { - this.tagKey = tagKey; - } - - public String getTagValue() { - return tagValue; - } - - public void setTagValue(String tagValue) { - this.tagValue = tagValue; - } - - public String getHostHeader() { - return hostHeader; - } - - public void setHostHeader(String hostHeader) { - this.hostHeader = hostHeader; - } - - public String getIamRole() { - return iamRole; - } - - public void setIamRole(String iamRole) { - this.iamRole = iamRole; - } - - public int getConnectionTimeoutSeconds() { - return connectionTimeoutSeconds; - } - - public void setConnectionTimeoutSeconds(int connectionTimeoutSeconds) { - this.connectionTimeoutSeconds = connectionTimeoutSeconds; - } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java new file mode 100644 index 000000000..a8515171b --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/AwsECSDiscoveryConfig.java @@ -0,0 +1,71 @@ +/** + * Copyright 2016 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.foxtrot.core.querystore.impl; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.Min; + +/** + * AWS ECS based cluster configuration. + * See: https://github.com/hazelcast/hazelcast-aws#ecsfargate-configuration + */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class AwsECSDiscoveryConfig extends ClusterDiscoveryConfig { + + @NotEmpty + @JsonProperty + private String network; + + @JsonProperty + private String accessKey; + + @JsonProperty + private String secretKey; + + @JsonProperty + private String region; + + @JsonProperty + private String cluster; + + @JsonProperty + private String family; + + @JsonProperty + private String serviceName; + + @JsonProperty + private String hostHeader; + + @JsonProperty + @Min(0) + private int opTimeoutSeconds; + + @JsonProperty + private boolean isExternalClient; + + public AwsECSDiscoveryConfig() { + super(ClusterDiscoveryType.FOXTROT_AWS_ECS); + } + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java index 60817757a..aacdcb8d4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ClusterDiscoveryType.java @@ -19,5 +19,6 @@ public enum ClusterDiscoveryType { FOXTROT_SIMPLE, FOXTROT_MARATHON, FOXTROT_AWS, + FOXTROT_AWS_ECS, FOXTROT_KUBERNETES } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java index def003ac1..a408e0dd4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchConfig.java @@ -16,6 +16,7 @@ package com.flipkart.foxtrot.core.querystore.impl; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; import lombok.NoArgsConstructor; import javax.validation.Valid; @@ -56,7 +57,8 @@ public enum ConnectionType { private long getQueryTimeout; private Integer port; @NotNull - private ConnectionType connectionType; + @Builder.Default + private ConnectionType connectionType = ConnectionType.HTTP; public List getHosts() { return hosts; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java index 633d95c3f..9ccb76303 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/ElasticsearchUtils.java @@ -17,6 +17,7 @@ import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.core.common.PeriodSelector; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -28,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.joda.time.DateTime; import org.joda.time.Interval; +import org.joda.time.LocalDate; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.slf4j.Logger; @@ -35,6 +37,7 @@ import java.io.IOException; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; /** * User: Santanu Sinha (santanu.sinha@flipkart.com) @@ -61,7 +64,8 @@ public class ElasticsearchUtils { private static final String MATCH_MAPPING_TYPE = "match_mapping_type"; private static String tableNamePrefix = "foxtrot"; - private ElasticsearchUtils() {} + private ElasticsearchUtils() { + } @VisibleForTesting public static String getTableNamePrefix() { @@ -69,9 +73,10 @@ public static String getTableNamePrefix() { } public static void setTableNamePrefix(ElasticsearchConfig config) { - if(StringUtils.isNotEmpty(config.getTableNamePrefix())) { + if (StringUtils.isNotEmpty(config.getTableNamePrefix())) { tableNamePrefix = config.getTableNamePrefix(); - } else { + } + else { tableNamePrefix = "foxtrot"; } } @@ -93,7 +98,7 @@ public static String[] getIndices(final String table, final ActionRequest reques DateTime start = interval.getStart() .toLocalDate() .toDateTimeAtStartOfDay(); - if(start.getYear() <= 1970) { + if (start.getYear() <= 1970) { logger.warn("Request of type {} running on all indices", request.getClass() .getSimpleName()); return new String[]{getIndices(table)}; @@ -116,7 +121,11 @@ public static String[] getIndices(final String table, final ActionRequest reques public static String getCurrentIndex(final String table, long timestamp) { //TODO::THROW IF TIMESTAMP IS BEYOND TABLE META.TTL String datePostfix = FORMATTER.print(timestamp); - return String.format("%s-%s-%s-%s", getTableNamePrefix(), table, ElasticsearchUtils.TABLENAME_POSTFIX, datePostfix); + return String.format("%s-%s-%s-%s", + getTableNamePrefix(), + table, + ElasticsearchUtils.TABLENAME_POSTFIX, + datePostfix); } public static PutIndexTemplateRequest getClusterTemplateMapping() { @@ -124,7 +133,8 @@ public static PutIndexTemplateRequest getClusterTemplateMapping() { return new PutIndexTemplateRequest().name("template_foxtrot_mappings") .patterns(Lists.newArrayList(String.format("%s-*", getTableNamePrefix()))) .mapping(DOCUMENT_TYPE_NAME, getDocumentMapping()); - } catch (IOException ex) { + } + catch (IOException ex) { logger.error("TEMPLATE_CREATION_FAILED", ex); return null; } @@ -253,8 +263,9 @@ public static void initializeMappings(RestHighLevelClient client) { public static String getValidTableName(String table) { - if(table == null) + if (table == null) { return null; + } return table.trim() .toLowerCase(); } @@ -265,7 +276,7 @@ private static boolean isIndexValidForTable(String index, String table) { } static boolean isIndexEligibleForDeletion(String index, Table table) { - if(index == null || table == null || !isIndexValidForTable(index, table.getName())) { + if (index == null || table == null || !isIndexValidForTable(index, table.getName())) { return false; } @@ -284,11 +295,12 @@ public static DateTime parseIndexDate(String index, String table) { } public static String getTableNameFromIndex(String currentIndex) { - if(currentIndex.contains(getTableNamePrefix()) && currentIndex.contains(TABLENAME_POSTFIX)) { + if (currentIndex.contains(getTableNamePrefix()) && currentIndex.contains(TABLENAME_POSTFIX)) { String tempIndex = currentIndex.substring(currentIndex.indexOf(getTableNamePrefix()) + getTableNamePrefix().length() + 1); int position = tempIndex.lastIndexOf(String.format("-%s", TABLENAME_POSTFIX)); return tempIndex.substring(0, position); - } else { + } + else { return null; } } @@ -296,4 +308,19 @@ public static String getTableNameFromIndex(String currentIndex) { static String getAllIndicesPattern() { return String.format("%s-*-%s-*", getTableNamePrefix(), ElasticsearchUtils.TABLENAME_POSTFIX); } + + public static String getTodayIndicesPattern() { + String datePostfix = FORMATTER.print(LocalDate.now()); + return String.format("%s-.*-%s-%s", getTableNamePrefix(), ElasticsearchUtils.TABLENAME_POSTFIX, datePostfix); + } + + public static boolean isTimeFilterPresent(List filters) { + AtomicBoolean timeFilterPresent = new AtomicBoolean(false); + filters.forEach(filter -> { + if (ElasticsearchUtils.TIME_FIELD.equals(filter.getField())) { + timeFilterPresent.set(true); + } + }); + return timeFilterPresent.get(); + } } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java index 392ecc71a..444424b62 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/impl/HazelcastConnection.java @@ -15,10 +15,12 @@ */ package com.flipkart.foxtrot.core.querystore.impl; +import com.google.common.base.Strings; +import com.hazelcast.client.properties.ClientProperty; import com.hazelcast.config.*; import com.hazelcast.core.Hazelcast; import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.spi.properties.GroupProperty; +import com.hazelcast.spi.properties.ClusterProperty; import com.marathon.hazelcast.servicediscovery.MarathonDiscoveryStrategyFactory; import io.dropwizard.lifecycle.Managed; import org.slf4j.Logger; @@ -48,8 +50,7 @@ public class HazelcastConnection implements Managed { @Inject public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostException { Config hzConfig = new Config(); - hzConfig.getGroupConfig() - .setName(clusterConfig.getName()); + hzConfig.setClusterName(clusterConfig.getName()); switch (clusterConfig.getDiscovery() .getType()) { case FOXTROT_SIMPLE: @@ -81,14 +82,12 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept String appId = marathonClusterDiscoveryConfig.getApp() .replace("/", "") .trim(); - hzConfig.getGroupConfig() - .setName("foxtrot"); - hzConfig.getGroupConfig() - .setPassword("foxtrot"); - hzConfig.setProperty(GroupProperty.DISCOVERY_SPI_ENABLED.getName(), "true"); - hzConfig.setProperty(GroupProperty.DISCOVERY_SPI_PUBLIC_IP_ENABLED.getName(), "true"); - hzConfig.setProperty(GroupProperty.SOCKET_CLIENT_BIND_ANY.getName(), "true"); - hzConfig.setProperty(GroupProperty.SOCKET_BIND_ANY.getName(), "true"); + hzConfig.setClusterName("foxtrot"); + hzConfig.setProperty("hazelcast.application.validation.token", "foxtrot"); + hzConfig.setProperty(ClientProperty.DISCOVERY_SPI_ENABLED.getName(), "true"); + hzConfig.setProperty(ClientProperty.DISCOVERY_SPI_PUBLIC_IP_ENABLED.getName(), "true"); + hzConfig.setProperty(ClusterProperty.SOCKET_CLIENT_BIND_ANY.getName(), "true"); + hzConfig.setProperty(ClusterProperty.SOCKET_BIND_ANY.getName(), "true"); NetworkConfig networkConfig = hzConfig.getNetworkConfig(); networkConfig.setPublicAddress(System.getenv("HOST") + ":" + System.getenv("PORT_5701")); @@ -107,8 +106,8 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept discoveryStrategyConfig.addProperty("port-index", marathonClusterDiscoveryConfig.getPortIndex()); discoveryConfig.addDiscoveryStrategyConfig(discoveryStrategyConfig); break; - case FOXTROT_AWS: - AwsClusterDiscoveryConfig awsClusterDiscoveryConfig = (AwsClusterDiscoveryConfig) clusterConfig.getDiscovery(); + case FOXTROT_AWS: { + AwsClusterDiscoveryConfig ec2Config = (AwsClusterDiscoveryConfig) clusterConfig.getDiscovery(); NetworkConfig hazelcastConfigNetworkConfig = hzConfig.getNetworkConfig(); JoinConfig hazelcastConfigNetworkConfigJoin = hazelcastConfigNetworkConfig.getJoin(); hazelcastConfigNetworkConfigJoin.getTcpIpConfig() @@ -116,19 +115,87 @@ public HazelcastConnection(ClusterConfig clusterConfig) throws UnknownHostExcept hazelcastConfigNetworkConfigJoin.getMulticastConfig() .setEnabled(false); AwsConfig awsConfig = new AwsConfig(); - awsConfig.setAccessKey(awsClusterDiscoveryConfig.getAccessKey()); - awsConfig.setConnectionTimeoutSeconds(awsClusterDiscoveryConfig.getConnectionTimeoutSeconds()); - awsConfig.setHostHeader(awsClusterDiscoveryConfig.getHostHeader()); - awsConfig.setIamRole(awsClusterDiscoveryConfig.getIamRole()); - awsConfig.setRegion(awsClusterDiscoveryConfig.getRegion()); - awsConfig.setSecurityGroupName(awsClusterDiscoveryConfig.getSecurityGroupName()); - awsConfig.setSecretKey(awsClusterDiscoveryConfig.getSecretKey()); - awsConfig.setTagKey(awsClusterDiscoveryConfig.getTagKey()); - awsConfig.setTagValue(awsClusterDiscoveryConfig.getTagValue()); + + if(!Strings.isNullOrEmpty(ec2Config.getServiceName())) { + awsConfig.setProperty("service-name", ec2Config.getServiceName()); + } + if(!Strings.isNullOrEmpty(ec2Config.getAccessKey())) { + awsConfig.setProperty("access-key", ec2Config.getAccessKey()); + } + if(!Strings.isNullOrEmpty(ec2Config.getSecretKey())) { + awsConfig.setProperty("secret-key", ec2Config.getSecretKey()); + } + if(!Strings.isNullOrEmpty(ec2Config.getIamRole())) { + awsConfig.setProperty("iam-role", ec2Config.getIamRole()); + } + if(!Strings.isNullOrEmpty(ec2Config.getRegion())) { + awsConfig.setProperty("region", ec2Config.getRegion()); + } + if(!Strings.isNullOrEmpty(ec2Config.getHostHeader())) { + awsConfig.setProperty("host-header", ec2Config.getHostHeader()); + } + if(!Strings.isNullOrEmpty(ec2Config.getSecurityGroupName())) { + awsConfig.setProperty("security-group-name", ec2Config.getSecurityGroupName()); + } + if(ec2Config.getOpTimeoutSeconds() > 0) { + awsConfig.setProperty("connection-timeout-seconds", + Integer.toString(ec2Config.getOpTimeoutSeconds())); + awsConfig.setProperty("read-timeout-seconds", + Integer.toString(ec2Config.getOpTimeoutSeconds())); + } + if(ec2Config.isExternalClient()) { + awsConfig.setProperty("use-public-ip", Boolean.TRUE.toString()); + } hazelcastConfigNetworkConfigJoin.setAwsConfig(awsConfig); hazelcastConfigNetworkConfigJoin.getAwsConfig() .setEnabled(true); break; + } + case FOXTROT_AWS_ECS: + AwsECSDiscoveryConfig ecsConfig = (AwsECSDiscoveryConfig) clusterConfig.getDiscovery(); + NetworkConfig hazelcastConfigNetworkConfig = hzConfig.getNetworkConfig(); +// JoinConfig hazelcastConfigNetworkConfigJoin = hazelcastConfigNetworkConfig.getJoin(); + hazelcastConfigNetworkConfig.getJoin() + .getMulticastConfig() + .setEnabled(false); + + hazelcastConfigNetworkConfig.getInterfaces() + .setEnabled(true) + .addInterface(ecsConfig.getNetwork()); + + AwsConfig awsConfig = new AwsConfig(); + awsConfig.setEnabled(true); + if(!Strings.isNullOrEmpty(ecsConfig.getAccessKey())) { + awsConfig.setProperty("access-key", ecsConfig.getAccessKey()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getSecretKey())) { + awsConfig.setProperty("secret-key", ecsConfig.getSecretKey()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getRegion())) { + awsConfig.setProperty("region", ecsConfig.getRegion()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getCluster())) { + awsConfig.setProperty("cluster", ecsConfig.getCluster()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getFamily())) { + awsConfig.setProperty("family", ecsConfig.getFamily()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getServiceName())) { + awsConfig.setProperty("service-name", ecsConfig.getServiceName()); + } + if(!Strings.isNullOrEmpty(ecsConfig.getHostHeader())) { + awsConfig.setProperty("host-header", ecsConfig.getHostHeader()); + } + if(ecsConfig.getOpTimeoutSeconds() > 0) { + awsConfig.setProperty("connection-timeout-seconds", + Integer.toString(ecsConfig.getOpTimeoutSeconds())); + awsConfig.setProperty("read-timeout-seconds", + Integer.toString(ecsConfig.getOpTimeoutSeconds())); + } + if(ecsConfig.isExternalClient()) { + awsConfig.setProperty("use-public-ip", Boolean.TRUE.toString()); + } + break; case FOXTROT_KUBERNETES: logger.info("Using Kubernetes"); JoinConfig kbConfig = hzConfig.getNetworkConfig().getJoin(); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteConfig.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteConfig.java new file mode 100644 index 000000000..2f75f7b04 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteConfig.java @@ -0,0 +1,20 @@ +package com.flipkart.foxtrot.core.reroute; + +import com.flipkart.foxtrot.core.jobs.BaseJobConfig; +import lombok.Data; + +/*** + Created by mudit.g on Feb, 2019 + ***/ +@Data +public class ClusterRerouteConfig extends BaseJobConfig { + + private static final String JOB_NAME = "ClusterReallocation"; + + private double thresholdShardCountPercentage = 20; + + @Override + public String getJobName() { + return JOB_NAME; + } +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java new file mode 100644 index 000000000..60f315953 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ClusterRerouteManager.java @@ -0,0 +1,172 @@ +package com.flipkart.foxtrot.core.reroute; + +import com.flipkart.foxtrot.common.Date; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; +import org.elasticsearch.index.shard.ShardId; +import org.joda.time.DateTime; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; + +/*** + Created by mudit.g on Sep, 2019 + ***/ +@Slf4j +@Singleton +@Order(20) +public class ClusterRerouteManager { + + private final ElasticsearchConnection connection; + private final ClusterRerouteConfig clusterRerouteConfig; + + @Inject + public ClusterRerouteManager(ElasticsearchConnection connection, ClusterRerouteConfig clusterRerouteConfig) { + this.connection = connection; + this.clusterRerouteConfig = clusterRerouteConfig; + } + + /*public void reallocate() { + Map nodeIdVsNodeInfoMap = new HashMap<>(); + BiMap nodeNameVsNodeId = HashBiMap.create(); + this.createNodeInfoMap(nodeIdVsNodeInfoMap); + this.createNodeNameVsNodeIdMap(nodeNameVsNodeId); + + int totalShards = getTotalShardCount(nodeIdVsNodeInfoMap); + double avgShardsPerNode = Math.ceil((double) totalShards / (double) nodeIdVsNodeInfoMap.size()); + double acceptableShardsPerNode = avgShardsPerNode + + Math.ceil((avgShardsPerNode * clusterRerouteConfig.getThresholdShardCountPercentage()) / 100); + Deque vacantNodeIds = getVacantNodeId((int) avgShardsPerNode, nodeIdVsNodeInfoMap); + + for (Map.Entry nodeIdVsNodeInfo : nodeIdVsNodeInfoMap.entrySet()) { + int shardCount = nodeIdVsNodeInfo.getValue().getShardInfos().size(); + if (shardCount > acceptableShardsPerNode) { + for (int i = shardCount; i >= (int) avgShardsPerNode; i--) { + ShardId shardId = nodeIdVsNodeInfo.getValue().getShardInfos().get(i - 1).getShardId(); + if (!vacantNodeIds.isEmpty()) { + reallocateShard(shardId, nodeIdVsNodeInfo.getKey(), vacantNodeIds.pop()); + } + } + } + } + } + + private boolean reallocateShard(ShardId shardId, String fromNode, String toNode) { + MoveAllocationCommand moveAllocationCommand = new MoveAllocationCommand(shardId.getIndexName(), + shardId.getId(), + fromNode, + toNode); + ClusterRerouteRequest clusterRerouteRequest = new ClusterRerouteRequest(); + clusterRerouteRequest.add(moveAllocationCommand); + try { + ClusterRerouteResponse clusterRerouteResponse = connection.getClient() + .admin() + .cluster() + .reroute(clusterRerouteRequest) + .actionGet(); + log.info(String.format("Reallocating Shard. From Node: %s To Node: %s", fromNode, toNode)); + Thread.sleep((new Date(DateTime.now()).getHourOfDay() + 1) * 4000L); + return clusterRerouteResponse.isAcknowledged(); + } + catch (Exception e) { + log.error(String.format("Error in Reallocating Shard. From Node: %s To Node: %s. Error Message: %s", + fromNode, + toNode, + e.getMessage()), e); + return false; + } + } + + private void createNodeInfoMap( + Map nodeIdVsNodeInfoMap) { + nodeIdVsNodeInfoMap.clear(); + IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); + indicesStatsRequest.all(); + IndicesStatsResponse indicesStatsResponse = connection.getClient() + .admin() + .indices() + .stats(indicesStatsRequest) + .actionGet(); + Arrays.stream(indicesStatsResponse.getShards()) + .forEach(shardStats -> { + if (shardStats.getShardRouting() + .shardId() + .getIndexName() + .matches(ElasticsearchUtils.getTodayIndicesPattern()) + && !shardStats.getShardRouting().relocating()) { + ShardId shardId = shardStats.getShardRouting() + .shardId(); + ShardInfo shardInfo = ShardInfo.builder() + .shardId(shardId) + .build(); + String nodeId = shardStats.getShardRouting() + .currentNodeId(); + if (nodeIdVsNodeInfoMap.containsKey(nodeId)) { + nodeIdVsNodeInfoMap.get(nodeId) + .getShardInfos() + .add(shardInfo); + } + else { + List shardInfoList = Lists.newArrayList(shardInfo); + NodeInfo nodeInfo = NodeInfo.builder() + .shardInfos(shardInfoList) + .build(); + nodeIdVsNodeInfoMap.put(nodeId, nodeInfo); + } + } + }); + } + + private void createNodeNameVsNodeIdMap( + BiMap nodeNameVsNodeId) { + nodeNameVsNodeId.clear(); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(); + nodesInfoRequest.all(); + NodesInfoResponse nodesInfoResponse = connection.getClient() + .admin() + .cluster() + .nodesInfo(nodesInfoRequest) + .actionGet(); + nodesInfoResponse.getNodes() + .forEach(nodeInfo -> nodeNameVsNodeId.put(nodeInfo.getNode() + .getName(), nodeInfo.getNode() + .getId())); + } + + private int getTotalShardCount( + Map nodeIdVsNodeInfoMap) { + int totalShards = 0; + for (NodeInfo nodeInfo : nodeIdVsNodeInfoMap.values()) { + totalShards += nodeInfo.getShardInfos().size(); + } + return totalShards; + } + + private Deque getVacantNodeId(int avgShardsPerNode, + Map nodeIdVsNodeInfoMap) { + Deque vacantNodeIds = new ArrayDeque<>(); + for (Map.Entry nodeIdVsNodeInfo : nodeIdVsNodeInfoMap.entrySet()) { + int shardCount = nodeIdVsNodeInfo.getValue().getShardInfos().size(); + if (shardCount < avgShardsPerNode) { + for (int i = avgShardsPerNode; i > shardCount; i--) { + vacantNodeIds.push(nodeIdVsNodeInfo.getKey()); + } + } + } + return vacantNodeIds; + }*/ + +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/NodeInfo.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/NodeInfo.java new file mode 100644 index 000000000..b7955fd65 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/NodeInfo.java @@ -0,0 +1,20 @@ +package com.flipkart.foxtrot.core.reroute; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/*** + Created by mudit.g on Feb, 2019 + ***/ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class NodeInfo { + + private List shardInfos; +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ShardInfo.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ShardInfo.java new file mode 100644 index 000000000..a01fdec98 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/reroute/ShardInfo.java @@ -0,0 +1,19 @@ +package com.flipkart.foxtrot.core.reroute; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.elasticsearch.index.shard.ShardId; + +/*** + Created by mudit.g on Feb, 2019 + ***/ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ShardInfo { + + private ShardId shardId; +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java index 43b0de418..0bf22bca7 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/DistributedTableMetadataManager.java @@ -42,12 +42,15 @@ import com.hazelcast.config.MapConfig; import com.hazelcast.config.MapStoreConfig; import com.hazelcast.config.NearCacheConfig; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import lombok.SneakyThrows; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.*; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.TimeValue; diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java index fe7633aa6..5aeecf194 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/FoxtrotTableManager.java @@ -71,6 +71,7 @@ public void update(Table table) { throw FoxtrotExceptions.createTableMissingException(table.getName()); } metadataManager.save(table); + dataStore.updateTable(table); } @Override diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java index 0913b34de..f9ab4ee40 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/table/impl/TableMapStore.java @@ -23,8 +23,8 @@ import com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.hazelcast.core.MapStore; -import com.hazelcast.core.MapStoreFactory; +import com.hazelcast.map.MapStore; +import com.hazelcast.map.MapStoreFactory; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.get.*; @@ -252,3 +252,4 @@ public TableMapStore newMapStore(String mapName, Properties properties) { } } } + diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/Constants.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/Constants.java new file mode 100644 index 000000000..a00d4e1a0 --- /dev/null +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/Constants.java @@ -0,0 +1,13 @@ +package com.flipkart.foxtrot.core.util; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +/*** + Created by nitish.goyal on 01/06/20 + ***/ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class Constants { + + public static final int CACHE_VALUE_SIZE_IN_KB = 256 * 1024; +} diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java index 49395709f..fa90d7231 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/util/ElasticsearchQueryUtils.java @@ -16,7 +16,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.common.ActionRequest; +import com.flipkart.foxtrot.common.query.Filter; +import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; +import com.google.common.collect.ImmutableList; +import lombok.val; +import org.elasticsearch.index.query.QueryBuilder; +import java.util.List; import java.util.Map; /*** @@ -34,4 +41,14 @@ public static Map toMap(ObjectMapper mapper, return mapper.convertValue(value, new TypeReference>() { }); } + + public static QueryBuilder translateFilter(ActionRequest request, List extraFilters) { + val filters = (null == extraFilters || extraFilters.isEmpty()) + ? request.getFilters() + : ImmutableList.builder() + .addAll(request.getFilters()) + .addAll(extraFilters) + .build(); + return new ElasticSearchQueryGenerator().genFilter(filters); + } } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java index 6b5c1fa6a..14eed3666 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/cacheable/DummyCacheableAction.java @@ -16,6 +16,7 @@ package com.flipkart.foxtrot.core.common.cacheable; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.core.common.Action; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.exception.MalformedQueryException; @@ -24,6 +25,8 @@ import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.action.ActionRequest; +import java.util.List; + /** * Created by rishabh.goyal on 02/05/14. */ @@ -57,7 +60,7 @@ public void validateImpl(DummyCacheableActionRequest parameter) throws Malformed } @Override - public ActionRequest getRequestBuilder(DummyCacheableActionRequest parameter) throws FoxtrotException { + public ActionRequest getRequestBuilder(DummyCacheableActionRequest parameter, List extraFilters) throws FoxtrotException { return null; } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java index 44de84424..690f3feea 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/common/noncacheable/NonCacheableAction.java @@ -16,6 +16,7 @@ package com.flipkart.foxtrot.core.common.noncacheable; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.core.common.Action; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.exception.MalformedQueryException; @@ -24,6 +25,8 @@ import com.google.common.annotations.VisibleForTesting; import org.elasticsearch.action.ActionRequest; +import java.util.List; + /** * Created by rishabh.goyal on 02/05/14. */ @@ -57,7 +60,7 @@ public void validateImpl(NonCacheableActionRequest parameter) throws MalformedQu } @Override - public ActionRequest getRequestBuilder(NonCacheableActionRequest parameter) throws FoxtrotException { + public ActionRequest getRequestBuilder(NonCacheableActionRequest parameter, List extraFilters) throws FoxtrotException { return null; } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/SeggregatedHBaseDataStoreTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/SeggregatedHBaseDataStoreTest.java index d4c5211e8..3f6239238 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/SeggregatedHBaseDataStoreTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/datastore/impl/hbase/SeggregatedHBaseDataStoreTest.java @@ -27,7 +27,7 @@ public class SeggregatedHBaseDataStoreTest { public void testSameTable() { HbaseConfig hbaseConfig = new HbaseConfig(); hbaseConfig.setTableName("foxtrot-test"); - Table table = new Table(TestUtils.TEST_TABLE_NAME, 7, false); + Table table = new Table(TestUtils.TEST_TABLE_NAME, 7, false, 1); Assert.assertEquals(hbaseConfig.getTableName(), TableUtil.getTableName(hbaseConfig, table)); } @@ -35,7 +35,7 @@ public void testSameTable() { public void testSegTable() { HbaseConfig hbaseConfig = new HbaseConfig(); hbaseConfig.setTableName("foxtrot-test"); - Table table = new Table(TestUtils.TEST_TABLE_NAME, 7, true); + Table table = new Table(TestUtils.TEST_TABLE_NAME, 7, true, 1); Assert.assertEquals(TestUtils.TEST_TABLE_NAME, TableUtil.getTableName(hbaseConfig, table)); } @@ -44,7 +44,7 @@ public void testPrefixedSegTable() { HbaseConfig hbaseConfig = new HbaseConfig(); hbaseConfig.setTableName("foxtrot-test"); hbaseConfig.setSeggregatedTablePrefix("foxtrot-"); - Table table = new Table(TestUtils.TEST_TABLE_NAME, 7, true); + Table table = new Table(TestUtils.TEST_TABLE_NAME, 7, true, 1); Assert.assertEquals(String.format("%s%s", hbaseConfig.getSeggregatedTablePrefix(), TestUtils.TEST_TABLE_NAME), TableUtil.getTableName(hbaseConfig, table) ); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java index b06d7e6b9..988e3ebb2 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/MultiQueryActionTest.java @@ -22,6 +22,7 @@ import com.flipkart.foxtrot.common.count.CountRequest; import com.flipkart.foxtrot.common.count.CountResponse; import com.flipkart.foxtrot.common.query.*; +import com.flipkart.foxtrot.common.query.general.EqualsFilter; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.google.common.collect.Maps; @@ -30,10 +31,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -50,6 +48,7 @@ public static void setUp() throws Exception { getElasticsearchConnection().getClient() .indices() .refresh(new RefreshRequest("*"), RequestOptions.DEFAULT); + Thread.sleep(2000); } @Test @@ -83,9 +82,47 @@ public void testMultiQuery() throws FoxtrotException, JsonProcessingException { CountResponse countResponse = (CountResponse)multiQueryResponse.getResponses() .get("2"); + System.out.println(getMapper().writerWithDefaultPrettyPrinter().writeValueAsString(queryResponse)); assertEquals(9, countResponse.getCount()); } + @Test + public void testMultiQueryCommonFilters() throws FoxtrotException, JsonProcessingException { + + HashMap requests = Maps.newHashMap(); + Query query = new Query(); + query.setTable(TestUtils.TEST_TABLE_NAME); + ResultSort resultSort = new ResultSort(); + resultSort.setOrder(ResultSort.Order.asc); + resultSort.setField("_timestamp"); + query.setSort(resultSort); + requests.put("1", query); + + CountRequest countRequest = new CountRequest(); + countRequest.setTable(TestUtils.TEST_TABLE_NAME); + countRequest.setField("os"); + countRequest.setDistinct(false); + requests.put("2", countRequest); + + MultiQueryRequest multiQueryRequest = new MultiQueryRequest(requests); + multiQueryRequest.setFilters(Collections.singletonList(new EqualsFilter("os", "ios"))); + + ActionResponse actionResponse = getQueryExecutor().execute(multiQueryRequest); + MultiQueryResponse multiQueryResponse = null; + if(actionResponse instanceof MultiQueryResponse) { + multiQueryResponse = (MultiQueryResponse)actionResponse; + } + assertNotNull(multiQueryResponse); + + QueryResponse queryResponse = (QueryResponse)multiQueryResponse.getResponses() + .get("1"); + CountResponse countResponse = (CountResponse)multiQueryResponse.getResponses() + .get("2"); + + assertEquals(2, queryResponse.getDocuments().size()); + assertEquals(2, countResponse.getCount()); + } + @Test public void testQueryNoFilterAscending() throws FoxtrotException, JsonProcessingException { HashMap requests = Maps.newHashMap(); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java index 42a39a85e..fdef73717 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/impl/DistributedTableMetadataManagerTest.java @@ -33,7 +33,7 @@ import com.google.common.collect.Lists; import com.hazelcast.config.Config; import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import com.hazelcast.test.TestHazelcastInstanceFactory; import org.joda.time.DateTime; import org.junit.*; diff --git a/foxtrot-server/pom.xml b/foxtrot-server/pom.xml index 20d35cc1d..24cdbb110 100644 --- a/foxtrot-server/pom.xml +++ b/foxtrot-server/pom.xml @@ -5,18 +5,27 @@ foxtrot com.flipkart.foxtrot - 6.3.1-8 + 6.3.1-9 4.0.0 foxtrot-server + + 0.7.0 + + io.dropwizard dropwizard-core ${dropwizard.version} + + io.dropwizard + dropwizard-auth + ${dropwizard.version} + ru.vyarus dropwizard-guicey @@ -48,7 +57,12 @@ - + + + org.bitbucket.b_c + jose4j + ${jose4j.version} + com.flipkart.foxtrot @@ -77,13 +91,44 @@ opencsv 2.0 + + com.google.api-client + google-api-client + 1.23.0 + + + io.appform.idman + idman-model + 1.0 + + + * + * + + + com.hazelcast hazelcast - 3.11.7 + ${hazelcast.version} test-jar test + + org.glassfish.jersey.test-framework.providers + jersey-test-framework-provider-grizzly2 + 2.25.1 + + + javax.servlet + javax.servlet-api + + + junit + junit + + + diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java new file mode 100644 index 000000000..cbeefc2f1 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/AuthInfoType.java @@ -0,0 +1,8 @@ +package com.flipkart.foxtrot.server; + +/** + * + */ +public enum AuthInfoType { + TOKEN +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java index 83990fa98..c530fbd19 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/FoxtrotServer.java @@ -19,13 +19,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; -import com.flipkart.foxtrot.core.querystore.impl.KubernetesClusterDiscoveryConfig; -import com.flipkart.foxtrot.core.querystore.impl.MarathonClusterDiscoveryConfig; -import com.flipkart.foxtrot.core.querystore.impl.SimpleClusterDiscoveryConfig; +import com.flipkart.foxtrot.core.querystore.impl.*; import com.flipkart.foxtrot.core.util.MetricUtil; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.di.FoxtrotModule; +import com.google.common.base.Strings; import com.google.inject.Stage; import io.dropwizard.Application; import io.dropwizard.assets.AssetsBundle; @@ -63,11 +61,20 @@ public boolean withOor() { } }); - final SwaggerBundleConfiguration swaggerBundleConfiguration = getSwaggerBundleConfiguration(); - bootstrap.addBundle(new SwaggerBundle() { @Override protected SwaggerBundleConfiguration getSwaggerBundleConfiguration(FoxtrotServerConfiguration configuration) { + final SwaggerBundleConfiguration swaggerBundleConfiguration = new SwaggerBundleConfiguration(); + swaggerBundleConfiguration.setTitle("Foxtrot"); + swaggerBundleConfiguration.setResourcePackage("com.flipkart.foxtrot.server.resources"); + swaggerBundleConfiguration.setUriPrefix("/foxtrot"); + swaggerBundleConfiguration.setDescription("A store abstraction and analytics system for real-time event data."); + if(!Strings.isNullOrEmpty(configuration.getSwaggerHost())) { + swaggerBundleConfiguration.setHost(configuration.getSwaggerHost()); + } + if(!Strings.isNullOrEmpty(configuration.getSwaggerScheme())) { + swaggerBundleConfiguration.setSchemes(new String[] {configuration.getSwaggerScheme()}); + } return swaggerBundleConfiguration; } }); @@ -115,15 +122,7 @@ private void configureObjectMapper(ObjectMapper objectMapper) { objectMapper.registerSubtypes(new NamedType(SimpleClusterDiscoveryConfig.class, "foxtrot_simple")); objectMapper.registerSubtypes(new NamedType(MarathonClusterDiscoveryConfig.class, "foxtrot_marathon")); objectMapper.registerSubtypes(new NamedType(KubernetesClusterDiscoveryConfig.class, "foxtrot_kubernetes")); - } - - private SwaggerBundleConfiguration getSwaggerBundleConfiguration() { - final SwaggerBundleConfiguration swaggerBundleConfiguration = new SwaggerBundleConfiguration(); - swaggerBundleConfiguration.setTitle("Foxtrot"); - swaggerBundleConfiguration.setResourcePackage("com.flipkart.foxtrot.server.resources"); - swaggerBundleConfiguration.setUriPrefix("/foxtrot"); - swaggerBundleConfiguration.setDescription("A store abstraction and analytics system for real-time event data."); - return swaggerBundleConfiguration; + objectMapper.registerSubtypes(new NamedType(AwsECSDiscoveryConfig.class, "foxtrot_aws_ecs")); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java index 81d94227a..684aae73b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/InitializerCommand.java @@ -29,11 +29,12 @@ import net.sourceforge.argparse4j.inf.Namespace; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.settings.Settings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,46 +48,63 @@ public InitializerCommand() { } @Override - protected void run(Bootstrap bootstrap, Namespace namespace, FoxtrotServerConfiguration configuration) + protected void run(Bootstrap bootstrap, + Namespace namespace, + FoxtrotServerConfiguration configuration) throws Exception { ElasticsearchConfig esConfig = configuration.getElasticsearch(); ElasticsearchConnection connection = new ElasticsearchConnection(esConfig); connection.start(); - ClusterHealthResponse clusterHealth = connection.getClient() - .cluster() - .health(new ClusterHealthRequest(), RequestOptions.DEFAULT); - int numDataNodes = clusterHealth.getNumberOfDataNodes(); - int numReplicas = (numDataNodes < 2) ? 0 : 1; - - logger.info("# data nodes: {}, Setting replica count to: {}", numDataNodes, numReplicas); + try { + ClusterHealthResponse clusterHealth = connection.getClient() + .cluster() + .health(new ClusterHealthRequest(), RequestOptions.DEFAULT); + int numDataNodes = clusterHealth.getNumberOfDataNodes(); + int numReplicas = (numDataNodes < 2) + ? 0 + : 1; - createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX, numReplicas); - createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_V2, numReplicas); - createMetaIndex(connection, TableMapStore.TABLE_META_INDEX, numReplicas); - createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_HISTORY, numReplicas); - createMetaIndex(connection, FqlStoreServiceImpl.FQL_STORE_INDEX, numReplicas); + logger.info("# data nodes: {}, Setting replica count to: {}", numDataNodes, numReplicas); - logger.info("Creating mapping"); - PutIndexTemplateRequest putIndexTemplateRequest = ElasticsearchUtils.getClusterTemplateMapping(); - AcknowledgedResponse response = connection.getClient() - .indices() - .putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); - logger.info("Created mapping: {}", response.isAcknowledged()); + createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX, numReplicas); + createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_V2, numReplicas); + createMetaIndex(connection, TableMapStore.TABLE_META_INDEX, numReplicas); + createMetaIndex(connection, ElasticsearchConsolePersistence.INDEX_HISTORY, numReplicas); + createMetaIndex(connection, FqlStoreServiceImpl.FQL_STORE_INDEX, numReplicas); + createMetaIndex(connection, "user-meta", numReplicas); + createMetaIndex(connection, "tokens", numReplicas); + logger.info("Creating mapping"); + PutIndexTemplateRequest putIndexTemplateRequest = ElasticsearchUtils.getClusterTemplateMapping(); + AcknowledgedResponse response = connection.getClient() + .indices() + .putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); + logger.info("Created mapping: {}", response.isAcknowledged()); + } + finally { + connection.stop(); + } logger.info("Creating hbase table"); - HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase() - .getTableName()); + HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase().getTableName()); + logger.info("Initialization complete..."); } private void createMetaIndex(final ElasticsearchConnection connection, final String indexName, int replicaCount) { try { + if(connection.getClient() + .indices() + .exists(new GetIndexRequest(indexName), RequestOptions.DEFAULT)) { + logger.info("Index {} already exists. Nothing to do.", indexName); + return; + } + logger.info("'{}' creation started", indexName); Settings settings = Settings.builder() .put("number_of_shards", 1) .put("number_of_replicas", replicaCount) .build(); - CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(indexName) + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName) .settings(settings); CreateIndexResponse response = connection.getClient() .indices() diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java new file mode 100644 index 000000000..6d238c2b5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthConfig.java @@ -0,0 +1,24 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.NoneAuthProviderConfig; +import lombok.Data; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +/** + * + */ +@Data +public class AuthConfig { + private boolean enabled; + + @NotNull + @Valid + private JwtConfig jwt = new JwtConfig(); + + @NotNull + @Valid + public AuthProviderConfig provider = new NoneAuthProviderConfig(); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java new file mode 100644 index 000000000..ecee575af --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfo.java @@ -0,0 +1,18 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.AuthInfoType; +import lombok.Data; + +/** + * + */ +@Data +public abstract class AuthInfo { + private final AuthInfoType type; + + protected AuthInfo(AuthInfoType type) { + this.type = type; + } + + public abstract T accept(final AuthInfoVisitor visitor); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java new file mode 100644 index 000000000..91dc17a6a --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthInfoVisitor.java @@ -0,0 +1,8 @@ +package com.flipkart.foxtrot.server.auth; + +/** + * + */ +public interface AuthInfoVisitor { + T visit(TokenAuthInfo tokenAuthInfo); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java new file mode 100644 index 000000000..96646c4ca --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthStore.java @@ -0,0 +1,73 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import io.dropwizard.util.Duration; +import lombok.SneakyThrows; +import lombok.val; + +import java.util.Date; +import java.util.HashSet; +import java.util.Optional; +import java.util.UUID; +import java.util.function.UnaryOperator; + +/** + * + */ +public interface AuthStore { + Optional provisionUser(final User user); + + Optional getUser(final String userId); + + boolean deleteUser(final String userId); + + boolean updateUser(final String userId, UnaryOperator mutator); + + default boolean grantRole(final String userId, final FoxtrotRole role) { + return updateUser(userId, user -> { + val roles = user.getRoles() == null ? new HashSet() : user.getRoles(); + roles.add(role); + return new User(userId, roles, user.getTables(), user.isSystemUser(), user.getCreated(), new Date()); + }); + } + + default boolean revokeRole(final String userId, final FoxtrotRole role) { + return updateUser(userId, user -> { + val roles = user.getRoles() == null ? new HashSet() : user.getRoles(); + roles.remove(role); + return new User(userId, roles, user.getTables(), user.isSystemUser(), user.getCreated(), new Date()); + }); + } + + default boolean grantTableAccess(final String userId, final String table) { + return updateUser(userId, user -> { + val tables = user.getTables() == null ? new HashSet() : user.getTables(); + tables.add(table); + return new User(userId, user.getRoles(), tables, user.isSystemUser(), user.getCreated(), new Date()); + }); + } + + default boolean revokeTableAccess(final String userId, final String table) { + return updateUser(userId, user -> { + val tables = user.getTables() == null ? new HashSet() : user.getTables(); + tables.remove(table); + return new User(userId, user.getRoles(), tables, user.isSystemUser(), user.getCreated(), new Date()); + }); + } + + default Optional provisionToken(final String userId, TokenType tokenType, Date expiry) { + return provisionToken(userId, UUID.randomUUID().toString(), tokenType, expiry); + } + + Optional provisionToken(final String userId, String tokenId, TokenType tokenType, Date expiry); + + Optional getToken(final String tokenId); + + @SneakyThrows + Optional getTokenForUser(String userId); + + boolean deleteToken(final String tokenId); + + boolean deleteExpiredTokens(Date date, Duration sessionDuration); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java new file mode 100644 index 000000000..5427ed187 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/AuthenticatedInfo.java @@ -0,0 +1,13 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.User; +import lombok.Value; + +/** + * + */ +@Value +public class AuthenticatedInfo { + Token token; + User user; +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java new file mode 100644 index 000000000..2c965df08 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/ESAuthStore.java @@ -0,0 +1,214 @@ +package com.flipkart.foxtrot.server.auth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.core.querystore.actions.Utils; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConnection; +import com.flipkart.foxtrot.server.auth.authprovider.IdType; +import io.dropwizard.util.Duration; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import javax.inject.Inject; +import java.util.Date; +import java.util.Optional; +import java.util.function.UnaryOperator; + +/** + * + */ +@Slf4j +public class ESAuthStore implements AuthStore { + public static final String USERS_INDEX = "user-meta"; + public static final String TOKENS_INDEX = "tokens"; + private static final String TOKEN_TYPE = "TOKEN"; + private static final String USER_TYPE = "USER"; + + private final ElasticsearchConnection connection; + private final ObjectMapper mapper; + + @Inject + public ESAuthStore(ElasticsearchConnection connection, ObjectMapper mapper) { + this.connection = connection; + this.mapper = mapper; + } + + @Override + @SneakyThrows + public Optional provisionUser(User user) { + val status = saveUser(user, DocWriteRequest.OpType.CREATE); + if (status != RestStatus.CREATED) { + return Optional.empty(); + } + return getUser(user.getId()); + } + + @Override + @SneakyThrows + public Optional getUser(String userId) { + val getResp = connection.getClient() + .get(new GetRequest(USERS_INDEX, USER_TYPE, userId), RequestOptions.DEFAULT); + if (!getResp.isExists()) { + return Optional.empty(); + } + return Optional.of(mapper.readValue(getResp.getSourceAsString(), User.class)); + } + + @SneakyThrows + @Override + public boolean deleteUser(String userId) { + boolean status = connection.getClient() + .delete(new DeleteRequest(USERS_INDEX, USER_TYPE, userId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) + .status() == RestStatus.OK; + if (status) { + val count = deleteTokensForUser(userId); + log.debug("User {} deleted and {} existing tokens deleted with that.", userId, count); + } + return status; + } + + @Override + @SneakyThrows + public boolean updateUser( + String userId, UnaryOperator mutator) { + val user = getUser(userId).orElse(null); + if (null == user) { + return false; + } + final User updatedUser = mutator.apply(user); + return saveUser(updatedUser, DocWriteRequest.OpType.INDEX) == RestStatus.OK; + } + + @Override + @SneakyThrows + public Optional provisionToken(String userId, String tokenId, TokenType tokenType, Date expiry) { + val user = getUser(userId).orElse(null); + if (null == user) { + log.warn("No user found for is: {}", userId); + return Optional.empty(); + } + if ((tokenType.equals(TokenType.STATIC) + || tokenType.equals(TokenType.SYSTEM)) + && !user.isSystemUser()) { + log.warn("Cannot create system/static token for non system user"); + return Optional.empty(); + } + try { + val saveStatus = connection.getClient() + .index(new IndexRequest(TOKENS_INDEX) + .source(mapper.writeValueAsString(new Token(tokenId, IdType.SESSION_ID, tokenType, userId, expiry)), + XContentType.JSON) + .id(tokenId) + .type(TOKEN_TYPE) + .create(true) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) + .status(); + if (saveStatus != RestStatus.CREATED) { + log.error("ES save status for token for user {} is: {}", userId, saveStatus); + return Optional.empty(); + } + log.info("Token created for user: {}", userId); + } + catch (ElasticsearchException v) { + log.warn("A valid token exists exists already.. for id: {}", userId); + return getTokenForUser(userId); + } + return getToken(tokenId); + } + + @Override + @SneakyThrows + public Optional getToken(String tokenId) { + val getResp = connection.getClient() + .get(new GetRequest(TOKENS_INDEX, TOKEN_TYPE, tokenId), RequestOptions.DEFAULT); + if (!getResp.isExists()) { + return Optional.empty(); + } + return Optional.of(mapper.readValue(getResp.getSourceAsString(), Token.class)); + } + + @Override + @SneakyThrows + public Optional getTokenForUser(String userId) { + val getResp = connection.getClient() + .search(new SearchRequest(TOKENS_INDEX) + .searchType(SearchType.QUERY_THEN_FETCH) + .types(TOKEN_TYPE) + .source(new SearchSourceBuilder().query( + QueryBuilders.termQuery("userId", userId))), + RequestOptions.DEFAULT); + final SearchHits hits = getResp.getHits(); + if (hits.totalHits <= 0) { + return Optional.empty(); + } + return Optional.of(mapper.readValue(hits.getAt(0).getSourceAsString(), Token.class)); + } + + @Override + @SneakyThrows + public boolean deleteToken(String tokenId) { + return connection.getClient() + .delete(new DeleteRequest(TOKENS_INDEX, TOKEN_TYPE, tokenId) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), RequestOptions.DEFAULT) + .status() == RestStatus.OK; + } + + @Override + @SneakyThrows + public boolean deleteExpiredTokens(Date date, Duration sessionDuration) { + log.info("Cleaning up sessions older than: {}", sessionDuration); + Date oldestValidDate = new Date(date.getTime() - sessionDuration.toMilliseconds()); + val deletedCount = connection.getClient() + .deleteByQuery(new DeleteByQueryRequest(TOKENS_INDEX) + .setDocTypes(TOKEN_TYPE) + .setIndicesOptions(Utils.indicesOptions()) + .setQuery(QueryBuilders.rangeQuery("expiry") + .lt(oldestValidDate.getTime())) + .setRefresh(true), RequestOptions.DEFAULT) + .getDeleted(); + log.info("Deleted {} expired tokens", deletedCount); + return true; + } + + + @SneakyThrows + private RestStatus saveUser(User user, DocWriteRequest.OpType opType) { + return connection.getClient() + .index(new IndexRequest(USERS_INDEX) + .source(mapper.writeValueAsString(user), XContentType.JSON) + .id(user.getId()) + .type(USER_TYPE) + .opType(opType) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE), + RequestOptions.DEFAULT) + .status(); + } + + @SneakyThrows + public long deleteTokensForUser(String userId) { + return connection.getClient() + .deleteByQuery(new DeleteByQueryRequest(TOKENS_INDEX) + .setDocTypes(TOKEN_TYPE) + .setQuery(QueryBuilders.termQuery("userId", userId)) + .setRefresh(true), + RequestOptions.DEFAULT) + .getDeleted(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java new file mode 100644 index 000000000..5af4c9e4a --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/IdmanAuthStore.java @@ -0,0 +1,86 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import io.dropwizard.util.Duration; + +import java.util.Date; +import java.util.Optional; +import java.util.function.UnaryOperator; + +/** + * + */ +public class IdmanAuthStore implements AuthStore { + @Override + public Optional provisionUser(User user) { + return Optional.empty(); + } + + @Override + public Optional getUser(String userId) { + return Optional.empty(); + } + + @Override + public boolean deleteUser(String userId) { + return false; + } + + @Override + public boolean updateUser( + String userId, UnaryOperator mutator) { + return false; + } + + @Override + public boolean grantRole(String userId, FoxtrotRole role) { + return AuthStore.super.grantRole(userId, role); + } + + @Override + public boolean revokeRole(String userId, FoxtrotRole role) { + return AuthStore.super.revokeRole(userId, role); + } + + @Override + public boolean grantTableAccess(String userId, String table) { + return AuthStore.super.grantTableAccess(userId, table); + } + + @Override + public boolean revokeTableAccess(String userId, String table) { + return AuthStore.super.revokeTableAccess(userId, table); + } + + @Override + public Optional provisionToken(String userId, TokenType tokenType, Date expiry) { + return AuthStore.super.provisionToken(userId, tokenType, expiry); + } + + @Override + public Optional provisionToken( + String userId, String tokenId, TokenType tokenType, Date expiry) { + return Optional.empty(); + } + + @Override + public Optional getToken(String tokenId) { + return Optional.empty(); + } + + @Override + public Optional getTokenForUser(String userId) { + return Optional.empty(); + } + + @Override + public boolean deleteToken(String tokenId) { + return false; + } + + @Override + public boolean deleteExpiredTokens(Date date, Duration sessionDuration) { + return false; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java new file mode 100644 index 000000000..59a3d7ee5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JWTAuthenticationFailure.java @@ -0,0 +1,9 @@ +package com.flipkart.foxtrot.server.auth; + +import java.io.IOException; + +/** + * + */ +public class JWTAuthenticationFailure extends IOException { +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java new file mode 100644 index 000000000..bd870082b --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/JwtConfig.java @@ -0,0 +1,36 @@ +package com.flipkart.foxtrot.server.auth; + +import com.google.common.annotations.VisibleForTesting; +import io.dropwizard.util.Duration; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; + +/** + * + */ +@Data +@NoArgsConstructor +public class JwtConfig { + @NotNull + @NotEmpty + private String privateKey = "useless_default_key"; + + @NotNull + @NotEmpty + private String issuerId = "foxtrot"; + + @NotEmpty + @NotNull + private String authCachePolicy = "maximumSize=10000, expireAfterAccess=10m"; + + private Duration sessionDuration = Duration.days(30); + + @VisibleForTesting + public JwtConfig(String privateKey, String issuerId) { + this.privateKey = privateKey; + this.issuerId = issuerId; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java new file mode 100644 index 000000000..82a2c5fb7 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/RoleAuthorizer.java @@ -0,0 +1,33 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import io.dropwizard.auth.Authorizer; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.inject.Singleton; + +/** + * + */ +@Singleton +@Slf4j +public class RoleAuthorizer implements Authorizer { + @Override + public boolean authorize(UserPrincipal userPrincipal, String role) { + val user = userPrincipal.getUser(); + val foxtrotRole = FoxtrotRole.valueOf(role); + + if(!user.getRoles().contains(foxtrotRole)) { + log.warn("User {} is trying to access unauthorized role: {}", user.getId(), role); + return false; + } + val token = userPrincipal.getToken(); + if(!token.getTokenType().getAllowedRoles().contains(foxtrotRole)) { + log.warn("User {} trying to access resource with role: {} with token of tye: {}", + user.getId(), role, token.getTokenType().name()); + return false; + } + return true; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java new file mode 100644 index 000000000..e7cab5300 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/Token.java @@ -0,0 +1,24 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.auth.authprovider.IdType; +import lombok.Value; + +import java.util.Date; + +/** + * + */ +@Value +public class Token { + public static final Token DEFAULT = new Token("__DEFAULT_TOKEN__", + IdType.SESSION_ID, + TokenType.SYSTEM, + "__DEFAULT__", + null); + + String id; + IdType idType; + TokenType tokenType; + String userId; + Date expiry; +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java new file mode 100644 index 000000000..a3caeca7e --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthInfo.java @@ -0,0 +1,27 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.AuthInfoType; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import lombok.Value; + +/** + * + */ +@Value +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class TokenAuthInfo extends AuthInfo { + + String token; + + public TokenAuthInfo(String token) { + super(AuthInfoType.TOKEN); + this.token = token; + } + + @Override + public T accept(AuthInfoVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java new file mode 100644 index 000000000..2416ef119 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenAuthenticator.java @@ -0,0 +1,51 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.google.common.base.Strings; +import io.dropwizard.auth.AuthenticationException; +import io.dropwizard.auth.Authenticator; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.Optional; + +/** + * Authenticator that will be run + */ +@Slf4j +@Singleton +public class TokenAuthenticator implements Authenticator { + + private final AuthConfig config; + private final Provider provider; + + @Inject + public TokenAuthenticator(AuthConfig config, Provider provider) { + this.config = config; + this.provider = provider; + } + + @Override + public Optional authenticate(String token) throws AuthenticationException { + if(!config.isEnabled()) { + log.debug("Authentication is disabled"); + return Optional.of(UserPrincipal.DEFAULT); + } + log.debug("Auth called"); + if(Strings.isNullOrEmpty(token)) { + log.warn("authentication_failed::empty token"); + return Optional.empty(); + } + val info = provider.get() + .authenticate(new TokenAuthInfo(token)) + .orElse(null); + if (info == null) { + log.warn("authentication_failed::token_validation_failed"); + return Optional.empty(); + } + return Optional.of(new UserPrincipal(info.getUser(), info.getToken())); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java new file mode 100644 index 000000000..7369e86ef --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/TokenType.java @@ -0,0 +1,23 @@ +package com.flipkart.foxtrot.server.auth; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import lombok.Getter; + +import java.util.EnumSet; +import java.util.Set; + +/** + * + */ +@Getter +public enum TokenType { + DYNAMIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.CONSOLE, FoxtrotRole.SYSADMIN)), + STATIC(EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.INGEST)), + SYSTEM(EnumSet.allOf(FoxtrotRole.class)); + + private final Set allowedRoles; + + TokenType(Set allowedRoles) { + this.allowedRoles = allowedRoles; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java new file mode 100644 index 000000000..b38e0ee0c --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/UserPrincipal.java @@ -0,0 +1,24 @@ +package com.flipkart.foxtrot.server.auth; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.flipkart.foxtrot.core.auth.User; +import lombok.Value; + +import java.security.Principal; + +/** + * + */ +@Value +public class UserPrincipal implements Principal { + public static final UserPrincipal DEFAULT = new UserPrincipal(User.DEFAULT, Token.DEFAULT); + + User user; + Token token; + + @Override + @JsonIgnore + public String getName() { + return user.getId(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java new file mode 100644 index 000000000..38684bedf --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthConfigVisitor.java @@ -0,0 +1,16 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.NoneAuthProviderConfig; + +/** + * + */ +public interface AuthConfigVisitor { + T visit(GoogleAuthProviderConfig googleAuthConfig); + + T visit(IdmanAuthProviderConfig idmanAuthProviderConfig); + + T visit(NoneAuthProviderConfig noneAuthProviderConfig); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java new file mode 100644 index 000000000..b4e911c32 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProvider.java @@ -0,0 +1,27 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.flipkart.foxtrot.server.auth.AuthInfo; +import com.flipkart.foxtrot.server.auth.AuthenticatedInfo; +import com.flipkart.foxtrot.server.auth.Token; + +import java.util.Optional; + +/** + * + */ +public interface AuthProvider { + + AuthType type(); + + String redirectionURL(String sessionId); + + default boolean isPreregistrationRequired() { + return true; + } + + Optional login(String authCode, String sessionId); + + Optional authenticate(AuthInfo authInfo); + + boolean logout(String sessionId); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java new file mode 100644 index 000000000..136d6e4f7 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderConfig.java @@ -0,0 +1,33 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.impl.IdmanAuthProviderConfig; +import lombok.Data; + +/** + * + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(name = "OAUTH_GOOGLE", value = GoogleAuthProviderConfig.class), + @JsonSubTypes.Type(name = "OAUTH_IDMAN", value = IdmanAuthProviderConfig.class), +}) +@Data +public abstract class AuthProviderConfig { + private final AuthType type; + + private boolean enabled; + + protected AuthProviderConfig(AuthType type) { + this.type = type; + } + + protected AuthProviderConfig(AuthType type, boolean enabled) { + this(type); + this.enabled = enabled; + } + + abstract public T accept(AuthConfigVisitor visitor); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java new file mode 100644 index 000000000..4dc8d4648 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthProviderFactory.java @@ -0,0 +1,13 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.google.inject.Injector; + +/** + * + */ +public interface AuthProviderFactory { + + AuthProvider build(ObjectMapper mapper, AuthStore authStore); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java new file mode 100644 index 000000000..913dc16bd --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/AuthType.java @@ -0,0 +1,10 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +/** + * + */ +public enum AuthType { + NONE, + OAUTH_GOOGLE, + OAUTH_IDMAN +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java new file mode 100644 index 000000000..a0505b1d7 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/ConfiguredAuthProviderFactory.java @@ -0,0 +1,38 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.auth.authprovider.impl.*; + +/** + * + */ +public class ConfiguredAuthProviderFactory implements AuthProviderFactory { + private final AuthConfig authConfig; + + public ConfiguredAuthProviderFactory(AuthConfig authConfig) { + this.authConfig = authConfig; + } + + @Override + public AuthProvider build(ObjectMapper mapper, AuthStore authStore) { + return authConfig.getProvider() + .accept(new AuthConfigVisitor() { + @Override + public AuthProvider visit(GoogleAuthProviderConfig googleAuthConfig) { + return new GoogleAuthProvider(googleAuthConfig, authConfig, mapper, authStore); + } + + @Override + public AuthProvider visit(IdmanAuthProviderConfig idmanAuthProviderConfig) { + return new IdmanAuthProvider(idmanAuthProviderConfig, mapper); + } + + @Override + public AuthProvider visit(NoneAuthProviderConfig noneAuthProviderConfig) { + return new NoneAuthProvider(); + } + }); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java new file mode 100644 index 000000000..dcb06a85a --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/IdType.java @@ -0,0 +1,9 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +/** + * + */ +public enum IdType { + SESSION_ID, + ACCESS_TOKEN +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java new file mode 100644 index 000000000..5aad4ced9 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/NoneAuthProvider.java @@ -0,0 +1,37 @@ +package com.flipkart.foxtrot.server.auth.authprovider; + +import com.flipkart.foxtrot.server.auth.AuthInfo; +import com.flipkart.foxtrot.server.auth.AuthenticatedInfo; +import com.flipkart.foxtrot.server.auth.Token; + +import java.util.Optional; + +/** + * + */ +public class NoneAuthProvider implements AuthProvider { + @Override + public AuthType type() { + return AuthType.NONE; + } + + @Override + public String redirectionURL(String sessionId) { + throw new IllegalStateException("Redirection called on NONE auth mode"); + } + + @Override + public Optional login(String authCode, String sessionId) { + return Optional.empty(); + } + + @Override + public Optional authenticate(AuthInfo authInfo) { + return Optional.empty(); + } + + @Override + public boolean logout(String sessionId) { + return true; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java new file mode 100644 index 000000000..ae2c91184 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProvider.java @@ -0,0 +1,235 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.server.auth.*; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleTokenResponse; +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; +import io.dropwizard.util.Duration; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.jose4j.jwa.AlgorithmConstraints; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jwt.MalformedClaimException; +import org.jose4j.jwt.consumer.InvalidJwtException; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtConsumerBuilder; +import org.jose4j.jwt.consumer.JwtContext; +import org.jose4j.keys.HmacKey; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Date; +import java.util.Optional; + +/** + * + */ +@Slf4j +public class GoogleAuthProvider implements AuthProvider { + + public static final String CALLBACK_PATH = "/foxtrot/oauth/callback"; + + private final HttpTransport transport; + private final GoogleAuthorizationCodeFlow authorizationCodeFlow; + private final String redirectionUrl; + private final AuthConfig authConfig; + private final GoogleAuthProviderConfig googleAuthConfig; + private final ObjectMapper mapper; + private final AuthStore credentialsStorage; + private final JwtConsumer consumer; + + @Inject + public GoogleAuthProvider( + GoogleAuthProviderConfig googleAuthConfig, + AuthConfig authConfig, ObjectMapper mapper, + AuthStore credentialsStorage) { + this.authConfig = authConfig; + final NetHttpTransport.Builder transportBuilder = new NetHttpTransport.Builder(); + Proxy proxy = Proxy.NO_PROXY; + if (googleAuthConfig.getProxyType() != null) { + switch (googleAuthConfig.getProxyType()) { + case DIRECT: + break; + case HTTP: { + Preconditions.checkArgument(!Strings.isNullOrEmpty(googleAuthConfig.getProxyHost())); + proxy = new Proxy(Proxy.Type.HTTP, + proxyAddress(googleAuthConfig)); + break; + } + case SOCKS: + Preconditions.checkArgument(!Strings.isNullOrEmpty(googleAuthConfig.getProxyHost())); + proxy = new Proxy(Proxy.Type.SOCKS, + proxyAddress(googleAuthConfig)); + break; + } + } + this.transport = transportBuilder.setProxy(proxy) + .build(); + this.authorizationCodeFlow = new GoogleAuthorizationCodeFlow.Builder( + transport, + new JacksonFactory(), + googleAuthConfig.getClientId(), + googleAuthConfig.getClientSecret(), + ImmutableSet.of("https://www.googleapis.com/auth/userinfo.email")) + .build(); + this.redirectionUrl = (googleAuthConfig.isSecureEndpoint() + ? "https" + : "http") + + "://" + + googleAuthConfig.getServer() + + CALLBACK_PATH; + this.googleAuthConfig = googleAuthConfig; + this.mapper = mapper; + this.credentialsStorage = credentialsStorage; + this.consumer = buildJwtConsumer(); + } + + @Override + public AuthType type() { + return AuthType.OAUTH_GOOGLE; + } + + @Override + public String redirectionURL(String sessionId) { + final String url = authorizationCodeFlow.newAuthorizationUrl() + .setState(sessionId) + .setRedirectUri(this.redirectionUrl) +// .setRedirectUri("http://localhost:8080/auth/google") + .build(); + return !Strings.isNullOrEmpty(googleAuthConfig.getLoginDomain()) + ? (url + "&hd=" + googleAuthConfig.getLoginDomain()) + : url; + } + + @Override + public Optional login(String authToken, String sessionId) { + if (Strings.isNullOrEmpty(authToken)) { + return Optional.empty(); + } + val googleAuthorizationCodeTokenRequest + = authorizationCodeFlow.newTokenRequest(authToken); + final String email; + try { + final GoogleTokenResponse tokenResponse = googleAuthorizationCodeTokenRequest + .setRedirectUri(this.redirectionUrl) + .execute(); + final Credential credential = authorizationCodeFlow.createAndStoreCredential(tokenResponse, null); + final HttpRequestFactory requestFactory = transport.createRequestFactory(credential); + // Make an authenticated request + final GenericUrl url = new GenericUrl("https://www.googleapis.com/oauth2/v1/userinfo"); + final HttpRequest request = requestFactory.buildGetRequest(url); + request.getHeaders().setContentType("application/json"); + final String jsonIdentity = request.execute().parseAsString(); + log.debug("Identity: {}", jsonIdentity); + email = mapper.readTree(jsonIdentity).get("email").asText(); + } + catch (IOException e) { + log.error("Error logging in using google:", e); + return Optional.empty(); + } + val user = credentialsStorage.getUser(email) + .orElse(null); + if (null == user) { + log.warn("No authorized user found for email: {}", email); + return Optional.empty(); + } + final Duration sessionDuration = AuthUtils.sessionDuration(authConfig); + return credentialsStorage.provisionToken(user.getId(), + sessionId, + TokenType.DYNAMIC, + new Date(new Date().getTime() + sessionDuration.toMilliseconds())); + } + + @Override + public Optional authenticate(AuthInfo authInfo) { + val jwt = authInfo.accept(TokenAuthInfo::getToken); + final JwtContext jwtContext; + try { + jwtContext = consumer.process(jwt); + } + catch (InvalidJwtException e) { + log.error("Jwt validation failure: ", e); + return Optional.empty(); + } + final String userId; + final String tokenId; + try { + val claims = jwtContext.getJwtClaims(); + userId = claims.getSubject(); + tokenId = claims.getJwtId(); + } + catch (MalformedClaimException e) { + log.error(String.format("exception in claim extraction %s", e.getMessage()), e); + return Optional.empty(); + } + log.debug("authentication_requested userId:{} tokenId:{}", userId, tokenId); + val token = credentialsStorage.getToken(tokenId).orElse(null); + if (token == null) { + log.warn("authentication_failed::invalid_session userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + if (!token.getUserId().equals(userId)) { + log.warn("authentication_failed::user_mismatch userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + val user = credentialsStorage.getUser(token.getUserId()).orElse(null); + if (null == user) { + log.warn("authentication_failed::invalid_user userId:{} tokenId:{}", userId, tokenId); + return Optional.empty(); + } + log.debug("authentication_success userId:{} tokenId:{}", userId, tokenId); + + return Optional.of(new AuthenticatedInfo(token, user)); + } + + @Override + public boolean logout(String sessionId) { + return credentialsStorage.deleteToken(sessionId); + } + + @Override + public boolean isPreregistrationRequired() { + return false; + } + + private JwtConsumer buildJwtConsumer() { + final JwtConfig jwtConfig = authConfig.getJwt(); + final byte[] secretKey = jwtConfig.getPrivateKey().getBytes(StandardCharsets.UTF_8); + return new JwtConsumerBuilder() + .setRequireIssuedAt() + .setRequireSubject() + .setExpectedIssuer(jwtConfig.getIssuerId()) + .setVerificationKey(new HmacKey(secretKey)) + .setJwsAlgorithmConstraints(new AlgorithmConstraints( + AlgorithmConstraints.ConstraintType.WHITELIST, + AlgorithmIdentifiers.HMAC_SHA512)) + .setExpectedAudience(Arrays.stream(TokenType.values()) + .map(TokenType::name) + .toArray(String[]::new)) + .build(); + + } + + private InetSocketAddress proxyAddress(GoogleAuthProviderConfig googleAuthConfig) { + return new InetSocketAddress(googleAuthConfig.getProxyHost(), + googleAuthConfig.getProxyPort()); + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java new file mode 100644 index 000000000..ba28df58d --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/GoogleAuthProviderConfig.java @@ -0,0 +1,75 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthConfigVisitor; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; +import java.net.Proxy; + +/** + * + */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class GoogleAuthProviderConfig extends AuthProviderConfig { + @NotEmpty + @NotNull + private String clientId; + + @NotEmpty + @NotNull + private String clientSecret; + + private String loginDomain; + + @NotNull + @NotEmpty + private String server; + + @NotNull + private boolean secureEndpoint; + + private Proxy.Type proxyType; + + private String proxyHost; + + private int proxyPort; + + @Builder + public GoogleAuthProviderConfig( + boolean enabled, + String clientId, + String clientSecret, + String loginDomain, + String server, + boolean secureEndpoint, + Proxy.Type proxyType, + String proxyHost, + int proxyPort) { + super(AuthType.OAUTH_GOOGLE, enabled); + this.clientId = clientId; + this.clientSecret = clientSecret; + this.loginDomain = loginDomain; + this.server = server; + this.secureEndpoint = secureEndpoint; + this.proxyType = proxyType; + this.proxyHost = proxyHost; + this.proxyPort = proxyPort; + } + + public GoogleAuthProviderConfig() { + super(AuthType.OAUTH_GOOGLE); + } + + @Override + public T accept(AuthConfigVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java new file mode 100644 index 000000000..1e231c8ff --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProvider.java @@ -0,0 +1,177 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.server.auth.*; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import com.flipkart.foxtrot.server.auth.authprovider.IdType; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import io.appform.idman.model.IdmanUser; +import io.appform.idman.model.TokenInfo; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.http.HttpStatus; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.*; +import java.util.function.Function; + +/** + * + */ +@Slf4j +public class IdmanAuthProvider implements AuthProvider { + private final CloseableHttpClient client = HttpClients.createDefault(); + private final IdmanAuthProviderConfig config; + private final ObjectMapper mapper; + + public IdmanAuthProvider(IdmanAuthProviderConfig config, ObjectMapper mapper) { + this.config = config; + this.mapper = mapper; + + } + + @Override + public AuthType type() { + return AuthType.OAUTH_IDMAN; + } + + @Override + @SneakyThrows + public String redirectionURL(String sessionId) { + return new URIBuilder(config.getIdmanEndpoint()) + .setPath("/apis/oauth2/authorize") + .addParameter("response_type", "code") + .addParameter("client_id", config.getClientId()) + .addParameter("state", sessionId) + .addParameter("redirect_uri", config.getServerEndpoint() + "/foxtrot/oauth/callback") + .build() + .toString(); + } + + @Override + @SneakyThrows + public Optional login(String authCode, String sessionId) { + return remoteTokenCall("authorization_code", authCode, + ti -> { + val idmanUser = ti.getUser(); + val expiry = Date.from(Instant.now().plusSeconds(ti.getExpiry())); + return Optional.of(new Token(ti.getAccessToken(), + IdType.ACCESS_TOKEN, + TokenType.DYNAMIC, + idmanUser.getUser().getId(), + expiry)); + }); + } + + @Override + @SneakyThrows + public Optional authenticate(AuthInfo authInfo) { + return remoteTokenCall( + "refresh_token", + authInfo.accept(TokenAuthInfo::getToken), + ti -> { + val idmanUser = ti.getUser(); + val expiry = Date.from(Instant.now().plusSeconds(ti.getExpiry())); + val remoteUser = idmanUser.getUser(); + val t = new Token(ti.getAccessToken(), + IdType.ACCESS_TOKEN, + TokenType.DYNAMIC, + remoteUser.getId(), + expiry); + val user = new User(remoteUser.getId(), + mapToFoxtrotRoles(idmanUser), + Collections.emptySet(), + isSystemUser(idmanUser), + new Date(), + new Date()); + return Optional.of(new AuthenticatedInfo(t, user)); + }); + } + + @Override + @SneakyThrows + public boolean logout(String sessionId) { + if(Strings.isNullOrEmpty(sessionId)) { + log.warn("Empty token send for logout"); + return true; + } + return remoteTokenRevokeCall(sessionId); + } + + private Set mapToFoxtrotRoles(final IdmanUser idmanUser) { + switch (idmanUser.getRole()) { + case "FOXTROT_ADMIN": { + return EnumSet.allOf(FoxtrotRole.class); + } + case "FOXTROT_HUMAN_USER": { + return EnumSet.of(FoxtrotRole.CONSOLE, FoxtrotRole.QUERY); + } + case "FOXTROT_SYSTEM_USER": { + return EnumSet.of(FoxtrotRole.QUERY, FoxtrotRole.INGEST); + } + default: + throw new IllegalStateException("Unexpected value: " + idmanUser.getRole()); + } + } + + private boolean isSystemUser(final IdmanUser user) { + return user.getRole().equals("FOXTROT_ADMIN"); + } + + private Optional remoteTokenCall( + String callMode, + String param, + Function> handler) throws URISyntaxException, IOException { + val post = new HttpPost(new URIBuilder(config.getIdmanEndpoint()) + .setPath("/apis/oauth2/token") + .build()); + val form = ImmutableList.builder() + .add(new BasicNameValuePair(callMode.equals("authorization_code") + ? "code" + : "refresh_token", param)) + .add(new BasicNameValuePair("client_id", config.getClientId())) + .add(new BasicNameValuePair("client_secret", config.getClientSecret())) + .add(new BasicNameValuePair("grant_type", callMode)) + .build(); + post.setEntity(new UrlEncodedFormEntity(form)); + try (val response = client.execute(post)) { + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + log.error("Error calling {}: {}", post.getURI().toString(), response.getStatusLine()); + return Optional.empty(); + } + val ti = mapper.readValue(EntityUtils.toByteArray(response.getEntity()), TokenInfo.class); + return handler.apply(ti); + } + } + + private boolean remoteTokenRevokeCall(String token) throws URISyntaxException, IOException { + val post = new HttpPost(new URIBuilder(config.getIdmanEndpoint()) + .setPath("/apis/oauth2/revoke") + .build()); + val params = ImmutableList.builder() + .add(new BasicNameValuePair("client_id", config.getClientId())) + .add(new BasicNameValuePair("client_secret", config.getClientSecret())) + .add(new BasicNameValuePair("token", token)) + .build(); + post.setEntity(new UrlEncodedFormEntity(params)); + try (val response = client.execute(post)) { + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK; + } + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java new file mode 100644 index 000000000..f16c9ea93 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/IdmanAuthProviderConfig.java @@ -0,0 +1,49 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthConfigVisitor; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.hibernate.validator.constraints.NotEmpty; + +/** + * + */ +@Data +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +public class IdmanAuthProviderConfig extends AuthProviderConfig { + @NotEmpty + private String idmanEndpoint; + @NotEmpty + private String clientId; + @NotEmpty + private String clientSecret; + @NotEmpty + public String serverEndpoint; + + public IdmanAuthProviderConfig() { + super(AuthType.OAUTH_IDMAN); + } + + @Builder + public IdmanAuthProviderConfig( + AuthType type, + boolean enabled, + String idmanEndpoint, + String clientId, + String clientSecret) { + super(type, enabled); + this.idmanEndpoint = idmanEndpoint; + this.clientId = clientId; + this.clientSecret = clientSecret; + } + + @Override + public T accept(AuthConfigVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java new file mode 100644 index 000000000..8715c36c1 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/authprovider/impl/NoneAuthProviderConfig.java @@ -0,0 +1,20 @@ +package com.flipkart.foxtrot.server.auth.authprovider.impl; + +import com.flipkart.foxtrot.server.auth.authprovider.AuthConfigVisitor; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProviderConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthType; + +/** + * + */ +public class NoneAuthProviderConfig extends AuthProviderConfig { + + public NoneAuthProviderConfig() { + super(AuthType.NONE); + } + + @Override + public T accept(AuthConfigVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java new file mode 100644 index 000000000..2c95782b9 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/JwtAuthDynamicFeature.java @@ -0,0 +1,30 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import io.dropwizard.auth.AuthDynamicFeature; +import io.dropwizard.auth.AuthValueFactoryProvider; +import io.dropwizard.auth.Authorizer; +import io.dropwizard.setup.Environment; +import org.glassfish.jersey.server.filter.RolesAllowedDynamicFeature; + +import javax.inject.Inject; +import javax.ws.rs.ext.Provider; + +/** + * + */ +@Provider +public class JwtAuthDynamicFeature extends AuthDynamicFeature { + @Inject + public JwtAuthDynamicFeature( + AuthConfig authConfig, + Authorizer authorizer, + Environment environment) { + super(new UserAuthorizationFilter(authConfig, authorizer)); + if(null != environment) { + environment.jersey().register(new AuthValueFactoryProvider.Binder<>(UserPrincipal.class)); + environment.jersey().register(RolesAllowedDynamicFeature.class); + } + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java new file mode 100644 index 000000000..d501c4bd5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/SessionUser.java @@ -0,0 +1,32 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import lombok.Builder; +import lombok.Data; +import lombok.val; + +import java.io.Serializable; + +/** + * + */ +@Data +@Builder +public class SessionUser implements Serializable { + private static final long serialVersionUID = -7917711435258380077L; + + private final UserPrincipal user; + + private static ThreadLocal currentUser = new ThreadLocal<>(); + + public static void put(UserPrincipal user) { + currentUser.set(user); + } + + public static UserPrincipal take() { + val user = currentUser.get(); + currentUser.remove(); + return user; + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java new file mode 100644 index 000000000..e54addf58 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthenticationFilter.java @@ -0,0 +1,131 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import io.dropwizard.auth.AuthenticationException; +import io.dropwizard.auth.Authenticator; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.annotation.Priority; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.servlet.*; +import javax.servlet.annotation.WebFilter; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Priorities; +import javax.ws.rs.core.HttpHeaders; +import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; +import java.util.Set; + +/** + * This filter validates the token + */ +@Priority(Priorities.AUTHENTICATION) +@WebFilter("/*") +@Slf4j +public class UserAuthenticationFilter implements Filter { + private static final Set WHITELISTED_PATTERNS = ImmutableSet.builder() + .add("/foxtrot/oauth") + .add("^/foxtrot/auth.*") + .build(); + private final AuthConfig authConfig; + private final Provider> authenticator; + + @Inject + public UserAuthenticationFilter( + AuthConfig authConfig, Provider> authenticator) { + this.authConfig = authConfig; + this.authenticator = authenticator; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void doFilter( + ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + if(!authConfig.isEnabled()) { + log.trace("Auth disabled"); + chain.doFilter(request, response); + return; + } + HttpServletRequest httpRequest = (HttpServletRequest) request; + HttpServletResponse httpResponse = (HttpServletResponse) response; + final String requestURI = httpRequest.getRequestURI(); + if(WHITELISTED_PATTERNS.stream().anyMatch(requestURI::startsWith)) { + chain.doFilter(request, response); + return; + } + val jwt = getTokenFromCookieOrHeader(httpRequest).orElse(null); + if(!Strings.isNullOrEmpty(jwt)) { + try { + val principal = authenticator.get() + .authenticate(jwt).orElse(null); + if(null != principal) { + SessionUser.put(principal); + chain.doFilter(request, response); + return; + } + else { + log.info("No principal "); + } + } + catch (AuthenticationException e) { + log.error("Jwt validation failure: ", e); + } + } + else { + log.debug("No token in request"); + } + val referrer = httpRequest.getHeader(org.apache.http.HttpHeaders.REFERER); + val source = Strings.isNullOrEmpty(referrer) ? requestURI : referrer; + httpResponse.addCookie(new Cookie("redirection", source)); + httpResponse.sendRedirect("/foxtrot/oauth/login"); + } + + @Override + public void destroy() { + + } + + private Optional getTokenFromCookieOrHeader(HttpServletRequest servletRequest) { + val tokenFromHeader = getTokenFromHeader(servletRequest); + return tokenFromHeader.isPresent() ? tokenFromHeader : getTokenFromCookie(servletRequest); + } + + private Optional getTokenFromHeader(HttpServletRequest servletRequest) { + val header = servletRequest.getHeader(HttpHeaders.AUTHORIZATION); + if (header != null) { + int space = header.indexOf(' '); + if (space > 0) { + final String method = header.substring(0, space); + if ("Bearer".equalsIgnoreCase(method)) { + final String rawToken = header.substring(space + 1); + return Optional.of(rawToken); + } + } + } + return Optional.empty(); + } + + private Optional getTokenFromCookie(HttpServletRequest request) { + val cookies = request.getCookies(); + if(null != cookies && cookies.length != 0) { + val token = Arrays.stream(cookies).filter(cookie -> cookie.getName().equals("token")).findAny().orElse(null); + if(null != token) { + return Optional.of(token.getValue()); + } + } + return Optional.empty(); + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java new file mode 100644 index 000000000..9c16c7991 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/filter/UserAuthorizationFilter.java @@ -0,0 +1,71 @@ +package com.flipkart.foxtrot.server.auth.filter; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.JWTAuthenticationFailure; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import io.dropwizard.auth.Authorizer; + +import javax.annotation.Priority; +import javax.ws.rs.Priorities; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.security.Principal; + +/** + * This filter assigns role to validated user + */ +@Priority(Priorities.AUTHENTICATION) +public class UserAuthorizationFilter implements ContainerRequestFilter { + + private final AuthConfig authConfig; + private final Authorizer authorizer; + + public UserAuthorizationFilter( + AuthConfig authConfig, + Authorizer authorizer) { + this.authConfig = authConfig; + this.authorizer = authorizer; + } + + @Override + public void filter(final ContainerRequestContext requestContext) throws IOException { + if(!authConfig.isEnabled()) { + updateContext(requestContext, UserPrincipal.DEFAULT); + return; + } + UserPrincipal principal = SessionUser.take(); + if(null != principal) { + updateContext(requestContext, principal); + return; + } + throw new JWTAuthenticationFailure(); + } + + private void updateContext(ContainerRequestContext requestContext, UserPrincipal principal) { + requestContext.setSecurityContext(new SecurityContext() { + + @Override + public Principal getUserPrincipal() { + return principal; + } + + @Override + public boolean isUserInRole(String role) { + return authorizer.authorize(principal, role); + } + + @Override + public boolean isSecure() { + return requestContext.getSecurityContext().isSecure(); + } + + @Override + public String getAuthenticationScheme() { + return SecurityContext.BASIC_AUTH; + } + + }); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java new file mode 100644 index 000000000..61024301f --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/io/CreateUserRequest.java @@ -0,0 +1,23 @@ +package com.flipkart.foxtrot.server.auth.io; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import lombok.Value; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.validation.constraints.NotNull; +import java.util.Set; + +/** + * + */ +@Value +public class CreateUserRequest { + @NotNull + @NotEmpty + String id; + @NotNull + @NotEmpty + Set roles; + Set tables; + boolean system; +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java new file mode 100644 index 000000000..57855a133 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/DistributedSessionDataStore.java @@ -0,0 +1,84 @@ +package com.flipkart.foxtrot.server.auth.sessionstore; + +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.google.common.base.Strings; +import com.hazelcast.config.InMemoryFormat; +import com.hazelcast.config.MapConfig; +import com.hazelcast.config.NearCacheConfig; +import com.hazelcast.map.IMap; +import io.dropwizard.lifecycle.Managed; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +/** + * + */ +@Singleton +@Order(50) +public class DistributedSessionDataStore implements SessionDataStore, Managed { + private static final String MAP_NAME = "REFERER_CACHE"; + private final HazelcastConnection hazelcastConnection; + + private IMap store; + + @Inject + public DistributedSessionDataStore(HazelcastConnection hazelcastConnection) { + this.hazelcastConnection = hazelcastConnection; + this.hazelcastConnection.getHazelcastConfig() + .getMapConfigs() + .put(MAP_NAME, mapConfig()); + } + + + @Override + public void put(String sessionId, Object data) { + if(Strings.isNullOrEmpty(sessionId) || null == data) { + return; + } + store.put(sessionId, data); + } + + @Override + public Optional get(String sessionId) { + if(Strings.isNullOrEmpty(sessionId)) { + return Optional.empty(); + } + return Optional.ofNullable(store.get(sessionId)); + } + + @Override + public void delete(String sessionId) { + if(Strings.isNullOrEmpty(sessionId)) { + return; + } + store.delete(sessionId); + } + + @Override + public void start() throws Exception { + this.store = this.hazelcastConnection.getHazelcast() + .getMap(MAP_NAME); + } + + @Override + public void stop() throws Exception { + //Do nothing + } + + private MapConfig mapConfig() { + MapConfig mapConfig = new MapConfig(); + mapConfig.setReadBackupData(true); + mapConfig.setInMemoryFormat(InMemoryFormat.BINARY); + mapConfig.setTimeToLiveSeconds(300); + mapConfig.setBackupCount(0); + + NearCacheConfig nearCacheConfig = new NearCacheConfig(); + nearCacheConfig.setTimeToLiveSeconds(300); + nearCacheConfig.setInvalidateOnChange(true); + mapConfig.setNearCacheConfig(nearCacheConfig); + return mapConfig; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java new file mode 100644 index 000000000..1c8e58c4f --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/auth/sessionstore/SessionDataStore.java @@ -0,0 +1,12 @@ +package com.flipkart.foxtrot.server.auth.sessionstore; + +import java.util.Optional; + +/** + * + */ +public interface SessionDataStore { + void put(String sessionId, Object data); + Optional get(String sessionId); + void delete(String sessionId); +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java index 6d451ff37..0d66fc318 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/cluster/ClusterManager.java @@ -6,9 +6,10 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.hazelcast.cluster.Member; import com.hazelcast.config.EvictionPolicy; import com.hazelcast.config.MapConfig; -import com.hazelcast.core.IMap; +import com.hazelcast.map.IMap; import io.dropwizard.lifecycle.Managed; import io.dropwizard.server.ServerFactory; import org.slf4j.Logger; @@ -46,7 +47,7 @@ public ClusterManager(HazelcastConnection connection, List healthCh mapConfig.setTimeToLiveSeconds(MAP_REFRESH_TIME + 2); //Reduce jitter mapConfig.setBackupCount(1); mapConfig.setAsyncBackupCount(2); - mapConfig.setEvictionPolicy(EvictionPolicy.NONE); + mapConfig.getEvictionConfig().setEvictionPolicy(EvictionPolicy.NONE); hazelcastConnection.getHazelcastConfig() .getMapConfigs() .put(MAP_NAME, mapConfig); @@ -80,6 +81,10 @@ public Collection getMembers() { return members.values(); } + public Collection getHazelcastMembers() { + return hazelcastConnection.getHazelcast().getCluster().getMembers(); + } + private static final class NodeDataUpdater implements Runnable { private final List healthChecks; private final ClusterMember clusterMember; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java index 790db31f6..1d95b16fb 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/FoxtrotServerConfiguration.java @@ -26,11 +26,13 @@ import com.flipkart.foxtrot.core.querystore.impl.CacheConfig; import com.flipkart.foxtrot.core.querystore.impl.ClusterConfig; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; +import com.flipkart.foxtrot.core.reroute.ClusterRerouteConfig; +import com.flipkart.foxtrot.server.auth.AuthConfig; import com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryConfig; +import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; import com.foxtrot.flipkart.translator.config.TranslatorConfig; import io.dropwizard.Configuration; -import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -63,29 +65,45 @@ public class FoxtrotServerConfiguration extends Configuration { private CardinalityConfig cardinality; @Valid private EsIndexOptimizationConfig esIndexOptimizationConfig; + + @Valid + private SessionCleanupConfig sessionCleanupConfig; + @Valid private ConsoleHistoryConfig consoleHistoryConfig; + @Valid private EmailConfig emailConfig; + @Valid private CacheConfig cacheConfig; + @Valid private RangerConfiguration rangerConfiguration; + @Valid private SegregationConfiguration segregationConfiguration; @NotNull private boolean restrictAccess; - private GandalfConfiguration gandalfConfiguration; - + @Valid private ElasticsearchTuningConfig elasticsearchTuningConfig; - @Builder.Default + @Valid + private String swaggerHost; + + private String swaggerScheme; + private TranslatorConfig translatorConfig = new TranslatorConfig(); @Valid - @Builder.Default private TextNodeRemoverConfiguration textNodeRemover = new TextNodeRemoverConfiguration(); + private ClusterRerouteConfig clusterRerouteConfig; + + @Valid + @NotNull + private AuthConfig auth = new AuthConfig(); + public FoxtrotServerConfiguration() { this.hbase = new HbaseConfig(); this.elasticsearch = new ElasticsearchConfig(); @@ -94,7 +112,7 @@ public FoxtrotServerConfiguration() { this.emailConfig = new EmailConfig(); this.segregationConfiguration = new SegregationConfiguration(); this.restrictAccess = true; - this.elasticsearchTuningConfig = new ElasticsearchTuningConfig(); + this.clusterRerouteConfig = new ClusterRerouteConfig(); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/GandalfConfiguration.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/GandalfConfiguration.java deleted file mode 100644 index 7cbca0ba8..000000000 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/config/GandalfConfiguration.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.flipkart.foxtrot.server.config; - -import lombok.Data; - -/*** - Created by nitish.goyal on 15/05/19 - ***/ -@Data -public class GandalfConfiguration { - - private String redirectUrl; - -} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java index 1238a49f4..cd36f6c7f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/di/FoxtrotModule.java @@ -35,25 +35,45 @@ import com.flipkart.foxtrot.core.table.TableMetadataManager; import com.flipkart.foxtrot.core.table.impl.DistributedTableMetadataManager; import com.flipkart.foxtrot.core.table.impl.FoxtrotTableManager; +import com.flipkart.foxtrot.server.auth.*; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.ConfiguredAuthProviderFactory; +import com.flipkart.foxtrot.server.auth.sessionstore.DistributedSessionDataStore; +import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; import com.flipkart.foxtrot.server.console.ConsolePersistence; import com.flipkart.foxtrot.server.console.ElasticsearchConsolePersistence; import com.flipkart.foxtrot.server.jobs.consolehistory.ConsoleHistoryConfig; +import com.flipkart.foxtrot.server.jobs.sessioncleanup.SessionCleanupConfig; import com.flipkart.foxtrot.sql.fqlstore.FqlStoreService; import com.flipkart.foxtrot.sql.fqlstore.FqlStoreServiceImpl; import com.foxtrot.flipkart.translator.config.SegregationConfiguration; import com.foxtrot.flipkart.translator.config.TranslatorConfig; +import com.google.common.cache.CacheBuilderSpec; import com.google.common.collect.ImmutableList; import com.google.inject.AbstractModule; +import com.google.inject.Injector; import com.google.inject.Provides; import com.google.inject.TypeLiteral; +import io.dropwizard.auth.Authenticator; +import io.dropwizard.auth.Authorizer; +import io.dropwizard.auth.CachingAuthenticator; +import io.dropwizard.auth.CachingAuthorizer; import io.dropwizard.server.ServerFactory; import io.dropwizard.setup.Environment; import io.dropwizard.util.Duration; +import lombok.val; import org.apache.hadoop.conf.Configuration; +import org.jose4j.jwa.AlgorithmConstraints; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jwt.consumer.JwtConsumer; +import org.jose4j.jwt.consumer.JwtConsumerBuilder; +import org.jose4j.keys.HmacKey; import javax.inject.Singleton; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -91,6 +111,10 @@ protected void configure() { .to(FoxtrotTableManager.class); bind(new TypeLiteral>() { }).toProvider(HealthcheckListProvider.class); + bind(AuthStore.class) + .to(ESAuthStore.class); + bind(SessionDataStore.class) + .to(DistributedSessionDataStore.class); } @Provides @@ -107,7 +131,7 @@ public ElasticsearchConfig esConfig(FoxtrotServerConfiguration configuration) { @Provides @Singleton - public TranslatorConfig getTranslatorConfig(FoxtrotServerConfiguration configuration){ + public TranslatorConfig getTranslatorConfig(FoxtrotServerConfiguration configuration) { return configuration.getTranslatorConfig(); } @@ -121,16 +145,16 @@ public ClusterConfig clusterConfig(FoxtrotServerConfiguration configuration) { @Singleton public CardinalityConfig cardinalityConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getCardinality() - ? new CardinalityConfig("false", String.valueOf(ElasticsearchUtils.DEFAULT_SUB_LIST_SIZE)) - : configuration.getCardinality(); + ? new CardinalityConfig("false", String.valueOf(ElasticsearchUtils.DEFAULT_SUB_LIST_SIZE)) + : configuration.getCardinality(); } @Provides @Singleton public EsIndexOptimizationConfig esIndexOptimizationConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getEsIndexOptimizationConfig() - ? new EsIndexOptimizationConfig() - : configuration.getEsIndexOptimizationConfig(); + ? new EsIndexOptimizationConfig() + : configuration.getEsIndexOptimizationConfig(); } @Provides @@ -143,8 +167,16 @@ public DataDeletionManagerConfig dataDeletionManagerConfig(FoxtrotServerConfigur @Singleton public ConsoleHistoryConfig consoleHistoryConfig(FoxtrotServerConfiguration configuration) { return null == configuration.getConsoleHistoryConfig() - ? new ConsoleHistoryConfig() - : configuration.getConsoleHistoryConfig(); + ? new ConsoleHistoryConfig() + : configuration.getConsoleHistoryConfig(); + } + + @Provides + @Singleton + public SessionCleanupConfig sessionCleanupConfig(FoxtrotServerConfiguration configuration) { + return null == configuration.getSessionCleanupConfig() + ? new SessionCleanupConfig() + : configuration.getSessionCleanupConfig(); } @Provides @@ -181,7 +213,9 @@ public List provideMutators( @Provides @Singleton - public List actionExecutionObservers(CacheManager cacheManager, InternalEventBus eventBus) { + public List actionExecutionObservers( + CacheManager cacheManager, + InternalEventBus eventBus) { return ImmutableList.builder() .add(new MetricRecorder()) .add(new ResponseCacheUpdater(cacheManager)) @@ -222,11 +256,93 @@ public ServerFactory serverFactory(FoxtrotServerConfiguration configuration) { return configuration.getServerFactory(); } + @Provides + @Singleton + public AuthConfig authConfig(FoxtrotServerConfiguration serverConfiguration) { + return serverConfiguration.getAuth(); + } + +/* @Provides + @Singleton + public GoogleAuthProviderConfig googleAuthProviderConfig(FoxtrotServerConfiguration configuration) { + return (GoogleAuthProviderConfig)configuration.getAuth().getProvider(); + }*/ + + @Provides + @Singleton + public AuthProvider authProvider( + FoxtrotServerConfiguration configuration, + AuthConfig authConfig, + Environment environment, + Injector injector) { + val authType = authConfig.getProvider().getType(); + AuthStore authStore = null; + switch (authType) { + case NONE: { + break; + } + case OAUTH_GOOGLE: + authStore = injector.getInstance(ESAuthStore.class); + break; + case OAUTH_IDMAN: + authStore = injector.getInstance(IdmanAuthStore.class); + break; + default: { + throw new IllegalArgumentException("Mode " + authType.name() + " not supported"); + } + } + return new ConfiguredAuthProviderFactory(configuration.getAuth()) + .build(environment.getObjectMapper(), authStore); + } + + @Provides + @Singleton + public JwtConsumer provideJwtConsumer(AuthConfig config) { + final JwtConfig jwtConfig = config.getJwt(); + final byte[] secretKey = jwtConfig.getPrivateKey().getBytes(StandardCharsets.UTF_8); + return new JwtConsumerBuilder() + .setRequireIssuedAt() + .setRequireSubject() + .setExpectedIssuer(jwtConfig.getIssuerId()) + .setVerificationKey(new HmacKey(secretKey)) + .setJwsAlgorithmConstraints(new AlgorithmConstraints( + AlgorithmConstraints.ConstraintType.WHITELIST, + AlgorithmIdentifiers.HMAC_SHA512)) + .setExpectedAudience(Arrays.stream(TokenType.values()) + .map(TokenType::name) + .toArray(String[]::new)) + .build(); + } + + @Provides + @Singleton + public Authenticator authenticator( + final Environment environment, + final TokenAuthenticator authenticator, + final AuthConfig authConfig) { + return new CachingAuthenticator<>( + environment.metrics(), + authenticator, + CacheBuilderSpec.parse(authConfig.getJwt().getAuthCachePolicy())); + } + + @Provides + @Singleton + public Authorizer authorizer( + final Environment environment, + final RoleAuthorizer authorizer, + final AuthConfig authConfig) { + return new CachingAuthorizer<>(environment.metrics(), + authorizer, + CacheBuilderSpec.parse(authConfig.getJwt().getAuthCachePolicy())); + } + @Provides @Singleton public ElasticsearchTuningConfig provideElasticsearchTuningConfig(FoxtrotServerConfiguration configuration) { return Objects.nonNull(configuration.getElasticsearchTuningConfig()) - ? configuration.getElasticsearchTuningConfig() : new ElasticsearchTuningConfig(); + ? configuration.getElasticsearchTuningConfig() + : new ElasticsearchTuningConfig(); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java new file mode 100644 index 000000000..03f8169b5 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/ExpiredSessionsCleaner.java @@ -0,0 +1,55 @@ +package com.flipkart.foxtrot.server.jobs.sessioncleanup; + +import com.flipkart.foxtrot.core.jobs.BaseJobManager; +import com.flipkart.foxtrot.core.querystore.impl.HazelcastConnection; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import lombok.extern.slf4j.Slf4j; +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockingTaskExecutor; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.time.Instant; +import java.util.Date; +import java.util.concurrent.ScheduledExecutorService; + +/** + * + */ +@Singleton +@Order(55) +@Slf4j +public class ExpiredSessionsCleaner extends BaseJobManager { + private final SessionCleanupConfig sessionCleanupConfig; + private final Provider authStore; + private final AuthConfig authConfig; + + @Inject + public ExpiredSessionsCleaner( + SessionCleanupConfig sessionCleanupConfig, + ScheduledExecutorService scheduledExecutorService, + HazelcastConnection hazelcastConnection, + Provider authStore, + AuthConfig authConfig) { + super(sessionCleanupConfig, scheduledExecutorService, hazelcastConnection); + this.sessionCleanupConfig = sessionCleanupConfig; + this.authStore = authStore; + this.authConfig = authConfig; + } + + @Override + protected void runImpl(LockingTaskExecutor executor, Instant lockAtMostUntil) { + executor.executeWithLock(() -> { + try { + authStore.get() + .deleteExpiredTokens(new Date(), AuthUtils.sessionDuration(authConfig)); + } catch (Exception e) { + log.error("Session cleanup failed: ", e); + } + }, new LockConfiguration(sessionCleanupConfig.getJobName(), lockAtMostUntil)); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java new file mode 100644 index 000000000..6ec775202 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/jobs/sessioncleanup/SessionCleanupConfig.java @@ -0,0 +1,15 @@ +package com.flipkart.foxtrot.server.jobs.sessioncleanup; + +import com.flipkart.foxtrot.core.jobs.BaseJobConfig; + +/** + * + */ +public class SessionCleanupConfig extends BaseJobConfig { + private static final String JOB_NAME = "ExpiredSessionCleaner"; + + @Override + public String getJobName() { + return JOB_NAME; + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java new file mode 100644 index 000000000..4c33442d7 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/providers/exception/AuthenticationExceptionHandler.java @@ -0,0 +1,25 @@ +package com.flipkart.foxtrot.server.providers.exception; + +import com.flipkart.foxtrot.server.auth.JWTAuthenticationFailure; +import lombok.extern.slf4j.Slf4j; + +import javax.inject.Singleton; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; +import java.net.URI; + +/** + * + */ +@Provider +@Singleton +@Slf4j +public class AuthenticationExceptionHandler implements ExceptionMapper { + + @Override + public Response toResponse(JWTAuthenticationFailure exception) { + log.error("Authentication failure: " + exception.getMessage(), exception); + return Response.seeOther(URI.create("/foxtrot/oauth/login")).build(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java index e45e1d4a5..418693d48 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AnalyticsResource.java @@ -20,14 +20,18 @@ import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionResponse; import com.flipkart.foxtrot.common.ActionValidationResponse; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.common.AsyncDataToken; import com.flipkart.foxtrot.core.querystore.QueryExecutor; +import com.flipkart.foxtrot.server.auth.UserPrincipal; +import io.dropwizard.auth.Auth; import com.flipkart.foxtrot.server.providers.FlatToCsvConverter; import com.flipkart.foxtrot.server.providers.FoxtrotExtraMediaType; import com.flipkart.foxtrot.sql.responseprocessors.Flattener; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.Valid; @@ -50,6 +54,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/analytics") @Singleton +@RolesAllowed(FoxtrotRole.Value.QUERY) public class AnalyticsResource { private final QueryExecutor queryExecutor; @@ -64,7 +69,10 @@ public AnalyticsResource(final QueryExecutor queryExecutor, final ObjectMapper o @POST @Timed @ApiOperation("runSync") - public ActionResponse runSync(@Valid final ActionRequest request) { + @RolesAllowed(FoxtrotRole.Value.QUERY) + public ActionResponse runSync( + @Auth final UserPrincipal userPrincipal, + @Valid final ActionRequest request) { return queryExecutor.execute(request); } @@ -72,7 +80,10 @@ public ActionResponse runSync(@Valid final ActionRequest request) { @Path("/async") @Timed @ApiOperation("runSyncAsync") - public AsyncDataToken runSyncAsync(@Valid final ActionRequest request) { + @RolesAllowed(FoxtrotRole.Value.QUERY) + public AsyncDataToken runSyncAsync( + @Auth final UserPrincipal userPrincipal, + @Valid final ActionRequest request) { return queryExecutor.executeAsync(request); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java index d8bee310f..4230334aa 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/AsyncResource.java @@ -17,11 +17,13 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.common.ActionResponse; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.cache.CacheManager; import com.flipkart.foxtrot.core.common.AsyncDataToken; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.constraints.NotNull; @@ -38,6 +40,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/async") @Singleton +@RolesAllowed(FoxtrotRole.Value.QUERY) public class AsyncResource { private CacheManager cacheManager; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java new file mode 100644 index 000000000..e9ece20e9 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/Auth.java @@ -0,0 +1,142 @@ +package com.flipkart.foxtrot.server.resources; + +import com.flipkart.foxtrot.core.auth.FoxtrotRole; +import com.flipkart.foxtrot.core.auth.User; +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.AuthStore; +import com.flipkart.foxtrot.server.auth.TokenType; +import com.flipkart.foxtrot.server.auth.io.CreateUserRequest; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import io.swagger.annotations.Api; +import lombok.val; +import org.hibernate.validator.constraints.NotEmpty; + +import javax.annotation.security.RolesAllowed; +import javax.inject.Inject; +import javax.inject.Provider; +import javax.validation.Valid; +import javax.validation.constraints.NotNull; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Collections; +import java.util.Date; + +/** + * + */ +@Path("/v1/auth") +@Produces(MediaType.APPLICATION_JSON) +@Api("Auth related APIs. DO NOT expose these for public access.") +@RolesAllowed(FoxtrotRole.Value.SYSADMIN) +public class Auth { + private final Provider authProvider; + private final AuthConfig authConfig; + + @Inject + public Auth(Provider authProvider, AuthConfig authConfig) { + this.authProvider = authProvider; + this.authConfig = authConfig; + } + + @POST + @Path("/users") + @Consumes(MediaType.APPLICATION_JSON) + public Response provisionUser(@NotNull @Valid final CreateUserRequest createUserRequest) { + val user = new User(createUserRequest.getId(), createUserRequest.getRoles(), createUserRequest.getTables(), + createUserRequest.isSystem(), new Date(), new Date()); + return Response.ok(authProvider.get().provisionUser(user)).build(); + } + + @GET + @Path("/users/{userId}") + public Response getUser(@NotNull @NotEmpty @PathParam("userId") final String userId) { + return Response.ok(authProvider.get().getUser(userId)).build(); + } + + @PUT + @Path("/users/{userId}/roles/grant/{role}") + public Response grantRole(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @PathParam("role") final FoxtrotRole role) { + val status = authProvider.get() + .grantRole(userId, role); + return updateUserResponse(userId, status); + } + + @PUT + @Path("/users/{userId}/roles/revoke/{role}") + public Response revokeRole(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @PathParam("role") final FoxtrotRole role) { + val status = authProvider.get() + .revokeRole(userId, role); + return updateUserResponse(userId, status); + } + + @PUT + @Path("/users/{userId}/tables/access/grant/{table}") + public Response grantTableAccess(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @NotEmpty @PathParam("table") final String table) { + val status = authProvider.get() + .grantTableAccess(userId, table); + return updateUserResponse(userId, status); + } + + @PUT + @Path("/users/{userId}/tables/access/revoke/{table}") + public Response revokeTableAccess(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @NotEmpty @PathParam("table") final String table) { + val status = authProvider.get() + .revokeTableAccess(userId, table); + return updateUserResponse(userId, status); + } + + @DELETE + @Path("/users/{userId}") + public Response deleteUser(@NotNull @NotEmpty @PathParam("userId") final String userId) { + final boolean status = authProvider.get().deleteUser(userId); + if(!status) { + return Response.notModified().build(); + } + return Response.ok().build(); + } + + @POST + @Path("/tokens/{userId}") + public Response provisionToken(@NotNull @NotEmpty @PathParam("userId") final String userId) { + val token = authProvider.get().provisionToken(userId, TokenType.STATIC, null).orElse(null); + if(null == token) { + return Response.notModified().build(); + } + return Response + .ok(Collections.singletonMap("jwt", AuthUtils.createJWT(token, authConfig.getJwt()))) + .build(); + } + + @GET + @Path("/tokens/{tokenId}") + public Response getToken(@NotNull @NotEmpty @PathParam("tokenId") final String tokenId) { + return Response.ok(authProvider.get().getToken(tokenId)) + .build(); + } + + @DELETE + @Path("/tokens/{userId}") + public Response deleteToken(@NotNull @NotEmpty @PathParam("userId") final String userId, + @NotNull @NotEmpty @PathParam("tokenId") final String tokenId) { + val status = authProvider.get().deleteToken(tokenId); + if(!status) { + return Response.notModified().build(); + } + return Response.ok().build(); + } + + private Response updateUserResponse(String userId, boolean status) { + if (!status) { + return Response.notModified() + .build(); + } + return Response.ok() + .entity(authProvider.get().getUser(userId)) + .build(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java index efd78437a..8afab1f6b 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/CacheUpdateResource.java @@ -20,6 +20,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.Consumes; @@ -38,6 +39,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/cache/update") @Singleton +@PermitAll public class CacheUpdateResource { private ExecutorService executorService; private TableMetadataManager tableMetadataManager; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java index 900b0c89f..8092e437a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterHealthResource.java @@ -19,11 +19,15 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.databind.JsonNode; import com.flipkart.foxtrot.core.querystore.QueryStore; +import com.flipkart.foxtrot.core.table.TableManager; +import com.flipkart.foxtrot.core.table.TableMetadataManager; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.SneakyThrows; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.GET; @@ -40,7 +44,10 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/clusterhealth") @Singleton +@PermitAll +@Order(20) public class ClusterHealthResource { + private final QueryStore queryStore; @Inject @@ -48,11 +55,10 @@ public ClusterHealthResource(QueryStore queryStore) { this.queryStore = queryStore; } - @GET @Timed @ApiOperation("getHealth") - public ClusterHealthResponse getHealth() throws ExecutionException, InterruptedException { + public ClusterHealthResponse getHealth() { return queryStore.getClusterHealth(); } @@ -60,7 +66,7 @@ public ClusterHealthResponse getHealth() throws ExecutionException, InterruptedE @Timed @Path("/nodestats") @ApiOperation("getNodeStat") - public JsonNode getNodeStat() throws ExecutionException, InterruptedException { + public JsonNode getNodeStat() { return queryStore.getNodeStats(); } @@ -69,7 +75,7 @@ public JsonNode getNodeStat() throws ExecutionException, InterruptedException { @Path("indicesstats") @ApiOperation("getIndicesStat") @SneakyThrows - public JsonNode getIndicesStat() throws ExecutionException, InterruptedException { + public JsonNode getIndicesStat() { return queryStore.getIndicesStats(); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java index f09528cb7..f7024ed2c 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ClusterInfoResource.java @@ -3,9 +3,11 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.server.cluster.ClusterManager; import com.flipkart.foxtrot.server.cluster.ClusterMember; +import com.hazelcast.cluster.Member; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.Consumes; @@ -22,6 +24,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/cluster") @Singleton +@PermitAll public class ClusterInfoResource { private ClusterManager clusterManager; @@ -38,4 +41,12 @@ public Map> members() { return Collections.singletonMap("members", clusterManager.getMembers()); } + + @GET + @Path("/hazelcast/members") + @Timed + @ApiOperation("Get Hazelcast cluster members") + public Map> hazelcastMembers() { + return Collections.singletonMap("members", clusterManager.getHazelcastMembers()); + } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java index 759edacb1..2de05c399 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleResource.java @@ -15,11 +15,13 @@ */ package com.flipkart.foxtrot.server.resources; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.server.console.Console; import com.flipkart.foxtrot.server.console.ConsolePersistence; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -31,6 +33,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/consoles") @Singleton +@RolesAllowed({FoxtrotRole.Value.QUERY, FoxtrotRole.Value.CONSOLE}) public class ConsoleResource { private ConsolePersistence consolePersistence; @@ -41,6 +44,7 @@ public ConsoleResource(ConsolePersistence consolePersistence) { } @POST + @RolesAllowed(FoxtrotRole.Value.CONSOLE) @ApiOperation("Save Console") public Console save(Console console) { consolePersistence.save(console); @@ -56,6 +60,7 @@ public Console get(@PathParam("id") final String id) { @DELETE @Path("/{id}/delete") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) @ApiOperation("Delete Console - via id") public void delete(@PathParam("id") final String id) { consolePersistence.delete(id); diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java index a320074bb..5a2d4f281 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ConsoleV2Resource.java @@ -16,11 +16,13 @@ package com.flipkart.foxtrot.server.resources; import com.codahale.metrics.annotation.Timed; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.server.console.ConsolePersistence; import com.flipkart.foxtrot.server.console.ConsoleV2; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -32,6 +34,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v2/consoles") @Singleton +@RolesAllowed({FoxtrotRole.Value.QUERY, FoxtrotRole.Value.CONSOLE}) public class ConsoleV2Resource { private ConsolePersistence consolePersistence; @@ -44,6 +47,7 @@ public ConsoleV2Resource(ConsolePersistence consolePersistence) { @POST @Timed @ApiOperation("Save Console") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public ConsoleV2 save(ConsoleV2 console) { consolePersistence.saveV2(console, true); return console; @@ -61,6 +65,7 @@ public ConsoleV2 get(@PathParam("id") final String id) { @Path("/{id}/delete") @Timed @ApiOperation("Delete Console - via id") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public void delete(@PathParam("id") final String id) { consolePersistence.deleteV2(id); } @@ -93,6 +98,7 @@ public List getOldVersions(@PathParam("name") final String name) { @Path("/{id}/old/delete") @Timed @ApiOperation("Delete old version console - via id") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public void deleteOldVersion(@PathParam("id") final String id) { consolePersistence.deleteOldVersion(id); } @@ -101,6 +107,7 @@ public void deleteOldVersion(@PathParam("id") final String id) { @Timed @Path("/{id}/old/set/current") @ApiOperation("Set old version console with id: {id} as current console") + @RolesAllowed(FoxtrotRole.Value.CONSOLE) public void setOldVersionAsCurrent(@PathParam("id") final String id) { consolePersistence.setOldVersionAsCurrent(id); } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java index 5f2a3b275..db94011e0 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/DocumentResource.java @@ -18,12 +18,14 @@ import com.codahale.metrics.annotation.Timed; import com.collections.CollectionUtils; import com.flipkart.foxtrot.common.Document; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.querystore.QueryStore; import com.foxtrot.flipkart.translator.TableTranslator; import com.google.common.collect.Lists; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.Valid; @@ -59,6 +61,7 @@ public DocumentResource(QueryStore queryStore, TableTranslator tableTranslator) @POST @Consumes(MediaType.APPLICATION_JSON) @Timed + @RolesAllowed(FoxtrotRole.Value.INGEST) @ApiOperation("Save Document") public Response saveDocument(@PathParam("table") String table, @Valid final Document document) { String tableName = tableTranslator.getTable(table, document); @@ -76,6 +79,7 @@ public Response saveDocument(@PathParam("table") String table, @Valid final Docu @Path("/bulk") @Consumes(MediaType.APPLICATION_JSON) @Timed + @RolesAllowed(FoxtrotRole.Value.INGEST) @ApiOperation("Save list of documents") public Response saveDocuments(@PathParam("table") String table, @Valid final List documents) { Map> tableVsDocuments = getTableVsDocuments(table, documents); @@ -89,6 +93,7 @@ public Response saveDocuments(@PathParam("table") String table, @Valid final Lis @GET @Path("/{id}") @Timed + @RolesAllowed(FoxtrotRole.Value.QUERY) @ApiOperation("Get Document") public Response getDocument(@PathParam("table") final String table, @PathParam("id") @NotNull final String id) { return Response.ok(queryStore.get(table, id)) @@ -97,6 +102,7 @@ public Response getDocument(@PathParam("table") final String table, @PathParam(" @GET @Timed + @RolesAllowed(FoxtrotRole.Value.QUERY) @ApiOperation("Get Documents") public Response getDocuments(@PathParam("table") final String table, @QueryParam("id") @NotNull final List ids) { return Response.ok(queryStore.getAll(table, ids)) diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java new file mode 100644 index 000000000..935bdbb73 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/ESClusterResource.java @@ -0,0 +1,40 @@ +package com.flipkart.foxtrot.server.resources; + +import com.flipkart.foxtrot.core.reroute.ClusterRerouteManager; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; + +/*** + Created by mudit.g on Sep, 2019 + ***/ +@Path("/v1/escluster") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Api(value = "/v1/escluster") +@Singleton +@Order(20) +public class ESClusterResource { + + private ClusterRerouteManager clusterRerouteManager; + + @Inject + public ESClusterResource(ClusterRerouteManager clusterRerouteManager) { + this.clusterRerouteManager = clusterRerouteManager; + } + + /*@GET + @Path("/reallocate") + @ApiOperation("reallocate shards") + public void reallocate() { + clusterRerouteManager.reallocate(); + }*/ +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java index 8d1928e6a..4bddc322f 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/FqlResource.java @@ -1,6 +1,7 @@ package com.flipkart.foxtrot.server.resources; import com.codahale.metrics.annotation.Timed; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.server.providers.FlatToCsvConverter; import com.flipkart.foxtrot.server.providers.FoxtrotExtraMediaType; import com.flipkart.foxtrot.sql.FqlEngine; @@ -12,6 +13,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -23,6 +25,7 @@ @Path("/v1/fql") @Api(value = "/v1/fql", description = "FQL API") @Singleton +@RolesAllowed(FoxtrotRole.Value.QUERY) public class FqlResource { private FqlEngine fqlEngine; private FqlStoreService fqlStoreService; diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/HbaseRegionsMergeResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/HbaseRegionsMergeResource.java new file mode 100644 index 000000000..a98a1a4df --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/HbaseRegionsMergeResource.java @@ -0,0 +1,59 @@ +package com.flipkart.foxtrot.server.resources; + +import com.codahale.metrics.annotation.Timed; +import com.flipkart.foxtrot.common.hbase.HRegionData; +import com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseConfig; +import com.flipkart.foxtrot.core.datastore.impl.hbase.HbaseRegions; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.apache.hadoop.hbase.TableName; +import ru.vyarus.dropwizard.guice.module.installer.order.Order; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.validation.constraints.Min; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@Path("/v1/hbase/regions") +@Produces(MediaType.APPLICATION_JSON) +@Api(value = "/v1/hbase/regions") +@Singleton +@Order(20) +public class HbaseRegionsMergeResource { + + private HbaseRegions hbaseRegions; + + @Inject + public HbaseRegionsMergeResource(HbaseConfig hbaseConfig) { + this.hbaseRegions = new HbaseRegions(hbaseConfig); + } + + @GET + @Path("/{table}/{threshSizeInGB}/list") + @Timed + @ApiOperation("Get all Hbase regions which can be merged") + public Map>> listMergableRegions(@PathParam("table") final String tableName, + @PathParam("threshSizeInGB") @Min(0) final double threshSizeInGB) { + return Collections.singletonMap("regions", hbaseRegions.getMergeableRegions(TableName.valueOf(tableName), threshSizeInGB)); + } + + @GET + @Path("/{table}/{threshSizeInGB}/merge/{number}") + @Timed + @ApiOperation("Merge Hbase regions") + public Response mergeRegions(@PathParam("table") final String tableName, + @PathParam("threshSizeInGB") @Min(0) final double threshSizeInGB, + @PathParam("number") @Min(-1) final int number) { + hbaseRegions.mergeRegions(TableName.valueOf(tableName), threshSizeInGB, number); + return Response.ok() + .build(); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/OAuth.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/OAuth.java new file mode 100644 index 000000000..b4c17048e --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/OAuth.java @@ -0,0 +1,159 @@ +package com.flipkart.foxtrot.server.resources; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.authprovider.AuthProvider; +import com.flipkart.foxtrot.server.auth.authprovider.IdType; +import com.flipkart.foxtrot.server.auth.authprovider.impl.GoogleAuthProvider; +import com.flipkart.foxtrot.server.auth.sessionstore.SessionDataStore; +import com.flipkart.foxtrot.server.utils.AuthUtils; +import com.google.common.base.Strings; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.http.HttpHeaders; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Cookie; +import javax.ws.rs.core.NewCookie; +import javax.ws.rs.core.Response; +import java.net.URI; +import java.util.UUID; + +/** + * + */ +@Path("/oauth") +@Slf4j +public class OAuth { + + private final AuthConfig authConfig; + private final Provider authProvider; + private final Provider sessionDataStore; + + @Inject + public OAuth( + AuthConfig authConfig, + Provider authProvider, + Provider sessionDataStore) { + this.authConfig = authConfig; + this.authProvider = authProvider; + this.sessionDataStore = sessionDataStore; + } + + @GET + @Path("/login") + public Response login( + @CookieParam("redirection") final Cookie cookieReferrer, + @HeaderParam(HttpHeaders.REFERER) final String referrer) { + final String sessionId = UUID.randomUUID().toString(); + final String redirectionURL = authProvider.get() + .redirectionURL(sessionId); + log.info("Redirection uri: {}", redirectionURL); + final String cookieReferrerUrl = null == cookieReferrer + ? null + : cookieReferrer.getValue(); + val source = Strings.isNullOrEmpty(cookieReferrerUrl) + ? referrer + : cookieReferrerUrl; + log.debug("Call source: {} Referrer: {} Redirection: {}", source, referrer, cookieReferrerUrl); + if (!Strings.isNullOrEmpty(source)) { + sessionDataStore.get() + .put(sessionId, source); + log.debug("Saved: {} against session: {}", source, sessionId); + } + return Response.seeOther(URI.create(redirectionURL)) + .cookie( + new NewCookie( + "gauth-state", + sessionId, + GoogleAuthProvider.CALLBACK_PATH, + null, + Cookie.DEFAULT_VERSION, + null, + NewCookie.DEFAULT_MAX_AGE, + null, + false, + false)) + .build(); + } + + @GET + @Path("/callback") + public Response handleGoogleCallback( + @CookieParam("gauth-state") final Cookie cookieState, + @Context HttpServletRequest requestContext, + @QueryParam("state") final String sessionId, + @QueryParam("code") final String authCode) { + log.info("Request Ctx: {}", requestContext); + if (null == cookieState + || !cookieState.getValue().equals(sessionId)) { + return Response.seeOther(URI.create("/")) + .cookie(expireCookie(cookieState)) + .build(); + } + val token = authProvider.get() + .login(authCode, sessionId) + .orElse(null); + if (null == token) { + return Response.seeOther(URI.create("/foxtrot/oauth/login")).build(); + } + val existingReferrer = sessionDataStore.get() + .get(token.getId()) + .orElse(null); + if (null != existingReferrer) { + sessionDataStore.get() + .delete(token.getId()); + } + log.debug("Got: {} against session: {}", existingReferrer, authCode); + final String finalRedirect = Strings.isNullOrEmpty((String) existingReferrer) + ? "/" + : (String) existingReferrer; + log.debug("Will be redirecting to: {}. Existing: {}", finalRedirect, existingReferrer); + return Response.seeOther(URI.create(finalRedirect)) + .cookie(createTokenCookie(token.getIdType().equals(IdType.SESSION_ID) + ? AuthUtils.createJWT(token, authConfig.getJwt()) + : token.getId()), + expireCookie(cookieState)) + .build(); + } + + @GET + @Path("/logout") +// @PermitAll + public Response logout(@CookieParam("token") final Cookie token) { + if (null != token) { + val jwt = token.getValue(); + log.debug("Token value: {}", jwt); + val status = authProvider.get().logout(jwt); + log.info("Token deletion status for token {} is {}", jwt, status); + return Response.seeOther(URI.create("/")) + .cookie(createTokenCookie("")) + .build(); + } + else { + log.info("Empty token in logout call"); + } + return Response.seeOther(URI.create("/")).build(); + } + + private NewCookie createTokenCookie(String value) { + return new NewCookie("token", + value, + "/", + null, + Cookie.DEFAULT_VERSION, + null, + NewCookie.DEFAULT_MAX_AGE, + null, + false, + true); + } + + private NewCookie expireCookie(@CookieParam("gauth-state") Cookie cookieState) { + return new NewCookie(cookieState, null, 0, null, false, true); + } + +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java index 2c30b1ebc..6b4b7b6aa 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableFieldMappingResource.java @@ -17,11 +17,14 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.table.TableManager; import com.flipkart.foxtrot.core.table.TableMetadataManager; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.*; @@ -36,6 +39,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/tables") @Singleton +@PermitAll public class TableFieldMappingResource { private final TableManager tableManager; @@ -80,6 +84,7 @@ public Response getAllFields(@QueryParam("withCardinality") @DefaultValue("false @Timed @Path("/{name}/fields/update") @ApiOperation("Update Fields") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) public Response updateEstimation(@PathParam("name") final String table, @QueryParam("time") @DefaultValue("0") long epoch) { tableMetadataManager.updateEstimationData(table, 0 == epoch ? System.currentTimeMillis() : epoch); return Response.ok() diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java index 4937745ec..13f952e0a 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/TableManagerResource.java @@ -17,11 +17,14 @@ import com.codahale.metrics.annotation.Timed; import com.flipkart.foxtrot.common.Table; +import com.flipkart.foxtrot.core.auth.FoxtrotRole; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.table.TableManager; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; +import javax.annotation.security.RolesAllowed; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.Valid; @@ -34,6 +37,7 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/tables") @Singleton +@PermitAll public class TableManagerResource { private final TableManager tableManager; @@ -43,16 +47,6 @@ public TableManagerResource(TableManager tableManager) { this.tableManager = tableManager; } - @POST - @Timed - @ApiOperation("Save Table") - public Response save(@Valid final Table table, @QueryParam("forceCreate") @DefaultValue("false") boolean forceCreate) { - table.setName(ElasticsearchUtils.getValidTableName(table.getName())); - tableManager.save(table, forceCreate); - return Response.ok(table) - .build(); - } - @GET @Timed @Path("/{name}") @@ -65,10 +59,32 @@ public Response get(@PathParam("name") String name) { .build(); } + @GET + @Timed + @ApiOperation("Get all Tables") + public Response getAll() { + return Response.ok() + .entity(tableManager.getAll()) + .build(); + } + + @POST + @Timed + @ApiOperation("Save Table") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) + public Response save(@Valid final Table table, @QueryParam("forceCreate") @DefaultValue("false") boolean forceCreate) { + table.setName(ElasticsearchUtils.getValidTableName(table.getName())); + tableManager.save(table, forceCreate); + return Response.ok(table) + .build(); + } + + @PUT @Timed @Path("/{name}") @ApiOperation("Update Table") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) public Response get(@PathParam("name") final String name, @Valid final Table table) { table.setName(name); tableManager.update(table); @@ -80,6 +96,7 @@ public Response get(@PathParam("name") final String name, @Valid final Table tab @Timed @Path("/{name}/delete") @ApiOperation("Delete Table") + @RolesAllowed(FoxtrotRole.Value.SYSADMIN) public Response delete(@PathParam("name") String name) { name = ElasticsearchUtils.getValidTableName(name); tableManager.delete(name); @@ -87,12 +104,4 @@ public Response delete(@PathParam("name") String name) { .build(); } - @GET - @Timed - @ApiOperation("Get all Tables") - public Response getAll() { - return Response.ok() - .entity(tableManager.getAll()) - .build(); - } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java index 658e815ef..9adc7ef55 100644 --- a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/resources/UtilResource.java @@ -16,10 +16,13 @@ package com.flipkart.foxtrot.server.resources; import com.codahale.metrics.annotation.Timed; -import com.flipkart.foxtrot.server.config.FoxtrotServerConfiguration; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchConfig; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import javax.annotation.security.PermitAll; import javax.inject.Inject; import javax.inject.Singleton; import javax.ws.rs.GET; @@ -31,19 +34,24 @@ @Produces(MediaType.APPLICATION_JSON) @Api(value = "/v1/util") @Singleton -public class UtilResource { - private final FoxtrotServerConfiguration configuration; +@PermitAll public class UtilResource { + private final ElasticsearchConfig elasticsearch; + private final ObjectMapper mapper; @Inject - public UtilResource(FoxtrotServerConfiguration configuration) { - this.configuration = configuration; + public UtilResource(ElasticsearchConfig elasticsearch, ObjectMapper mapper) { + this.elasticsearch = elasticsearch; + this.mapper = mapper; } @GET @Path("/config") @Timed @ApiOperation("Get config") - public FoxtrotServerConfiguration configuration() { - return configuration; + public JsonNode configuration() { + return mapper.createObjectNode() + .set("elasticsearch", mapper.createObjectNode() + .put("tableNamePrefix", elasticsearch.getTableNamePrefix()) + .set("hosts", mapper.valueToTree(elasticsearch.getHosts()))); } } diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java new file mode 100644 index 000000000..eba410bda --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/AuthUtils.java @@ -0,0 +1,50 @@ +package com.flipkart.foxtrot.server.utils; + +import com.flipkart.foxtrot.server.auth.AuthConfig; +import com.flipkart.foxtrot.server.auth.JwtConfig; +import com.flipkart.foxtrot.server.auth.Token; +import io.dropwizard.util.Duration; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.SneakyThrows; +import org.jose4j.jws.AlgorithmIdentifiers; +import org.jose4j.jws.JsonWebSignature; +import org.jose4j.jwt.JwtClaims; +import org.jose4j.jwt.NumericDate; +import org.jose4j.keys.HmacKey; + +import java.nio.charset.StandardCharsets; + +/** + * + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class AuthUtils { + @SneakyThrows + public static String createJWT(final Token token, final JwtConfig jwtConfig) { + JwtClaims claims = new JwtClaims(); + claims.setIssuer(jwtConfig.getIssuerId()); + claims.setGeneratedJwtId(); + claims.setIssuedAtToNow(); + claims.setJwtId(token.getId()); + claims.setNotBeforeMinutesInThePast(2); + claims.setSubject(token.getUserId()); + claims.setAudience(token.getTokenType().name()); + if(null != token.getExpiry()) { + claims.setExpirationTime(NumericDate.fromMilliseconds(token.getExpiry().getTime())); + } + JsonWebSignature jws = new JsonWebSignature(); + jws.setPayload(claims.toJson()); + final byte[] secretKey = jwtConfig.getPrivateKey().getBytes(StandardCharsets.UTF_8); + jws.setKey(new HmacKey(secretKey)); + jws.setAlgorithmHeaderValue(AlgorithmIdentifiers.HMAC_SHA512); + return jws.getCompactSerialization(); + } + + public static Duration sessionDuration(AuthConfig authConfig) { + final Duration dynamicSessionDuration = authConfig.getJwt().getSessionDuration(); + return dynamicSessionDuration != null + ? dynamicSessionDuration + : Duration.days(30); + } +} diff --git a/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/response/FoxtrotIndicesStatsResponse.java b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/response/FoxtrotIndicesStatsResponse.java new file mode 100644 index 000000000..78a0468d7 --- /dev/null +++ b/foxtrot-server/src/main/java/com/flipkart/foxtrot/server/utils/response/FoxtrotIndicesStatsResponse.java @@ -0,0 +1,20 @@ +package com.flipkart.foxtrot.server.utils.response; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; + +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class FoxtrotIndicesStatsResponse { + + private IndicesStatsResponse indicesStatsResponse; + private Map tableColumnCount; + +} diff --git a/foxtrot-server/src/main/resources/console/echo/browse-events.htm b/foxtrot-server/src/main/resources/console/echo/browse-events.htm index 3aaec5fdd..0c144a1bb 100644 --- a/foxtrot-server/src/main/resources/console/echo/browse-events.htm +++ b/foxtrot-server/src/main/resources/console/echo/browse-events.htm @@ -61,9 +61,12 @@

diff --git a/foxtrot-server/src/main/resources/console/echo/cluster/index.htm b/foxtrot-server/src/main/resources/console/echo/cluster/index.htm index 0cacecf43..3b63e518b 100644 --- a/foxtrot-server/src/main/resources/console/echo/cluster/index.htm +++ b/foxtrot-server/src/main/resources/console/echo/cluster/index.htm @@ -159,6 +159,8 @@

# Days Events Indexed Size + Average event size + Number of columns @@ -168,6 +170,8 @@

{{days}} {{events}} {{size}} + {{avgSize}} + {{columnCount}} {{/each}} diff --git a/foxtrot-server/src/main/resources/console/echo/css/foxtrot.css b/foxtrot-server/src/main/resources/console/echo/css/foxtrot.css index 2bb50e02b..ecc0bb155 100644 --- a/foxtrot-server/src/main/resources/console/echo/css/foxtrot.css +++ b/foxtrot-server/src/main/resources/console/echo/css/foxtrot.css @@ -3140,7 +3140,7 @@ div.tab button:first-child { position: relative; top: 20px; float: right; - right: 14px; + right: 24px; } #left-action-btns { diff --git a/foxtrot-server/src/main/resources/console/echo/fql/index.htm b/foxtrot-server/src/main/resources/console/echo/fql/index.htm index a8839a722..f0f4d42f0 100644 --- a/foxtrot-server/src/main/resources/console/echo/fql/index.htm +++ b/foxtrot-server/src/main/resources/console/echo/fql/index.htm @@ -87,10 +87,12 @@

diff --git a/foxtrot-server/src/main/resources/console/echo/index.htm b/foxtrot-server/src/main/resources/console/echo/index.htm index 78f92d2f4..a3c9ff03b 100644 --- a/foxtrot-server/src/main/resources/console/echo/index.htm +++ b/foxtrot-server/src/main/resources/console/echo/index.htm @@ -68,6 +68,7 @@

  • Create New Dashboard
    • +
  • Logout
    • @@ -298,9 +299,6 @@

    Refresh Time