Version 1.5.0 of the Amazon Kinesis Client Library

META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Amazon Kinesis Client Library for Java
 Bundle-SymbolicName: com.amazonaws.kinesisclientlibrary;singleton:=true
-Bundle-Version: 1.4.0
+Bundle-Version: 1.5.0
 Bundle-Vendor: Amazon Technologies, Inc
 Bundle-RequiredExecutionEnvironment: JavaSE-1.7
 Require-Bundle: org.apache.commons.codec;bundle-version="1.6",

NOTICE.txt

@@ -1,3 +1,3 @@
 AmazonKinesisClientLibrary
-Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 

README.md

@@ -29,6 +29,14 @@ For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesi
 To make it easier for developers to write record processors in other languages, we have implemented a Java based daemon, called MultiLangDaemon that does all the heavy lifting. Our approach has the daemon spawn a sub-process, which in turn runs the record processor, which can be written in any language. The MultiLangDaemon process and the record processor sub-process communicate with each other over [STDIN and STDOUT using a defined protocol][multi-lang-protocol]. There will be a one to one correspondence amongst record processors, child processes, and shards. For Python developers specifically, we have abstracted these implementation details away and [expose an interface][kclpy] that enables you to focus on writing record processing logic in Python. This approach enables KCL to be language agnostic, while providing identical features and similar parallel processing model across all languages.
 
 ## Release Notes
+### Release 1.5.0 (July 9, 2015)
+* **[Metrics Enhancements][kinesis-guide-monitoring-with-kcl]**
+	* Support metrics level and dimension configurations to control CloudWatch metrics emitted by the KCL.
+	* Add new metrics that track time spent in record processor methods.
+	* Disable WorkerIdentifier dimension by default.
+* **Exception Reporting** — Do not silently ignore exceptions in ShardConsumer.
+* **AWS SDK Component Dependencies** — Depend only on AWS SDK components that are used.
+
 ### Release 1.4.0 (June 2, 2015)
 * Integration with the **[Kinesis Producer Library (KPL)][kinesis-guide-kpl]**
 	* Automatically de-aggregate records put into the Kinesis stream using the KPL.
@@ -40,10 +48,10 @@ To make it easier for developers to write record processors in other languages,
 * A new metric called "MillisBehindLatest", which tracks how far consumers are from real time, is now uploaded to CloudWatch.
 
 ### Release 1.2.1 (January 26, 2015)
-* **MultiLangDaemon** Changes to the MultiLangDaemon to make it easier to provide a custom worker.
+* **MultiLangDaemon** — Changes to the MultiLangDaemon to make it easier to provide a custom worker.
 
 ### Release 1.2 (October 21, 2014)
-* **Multi-Language Support** Amazon KCL now supports implementing record processors in any language by communicating with the daemon over [STDIN and STDOUT][multi-lang-protocol]. Python developers can directly use the [Amazon Kinesis Client Library for Python][kclpy] to write their data processing applications.
+* **Multi-Language Support** — Amazon KCL now supports implementing record processors in any language by communicating with the daemon over [STDIN and STDOUT][multi-lang-protocol]. Python developers can directly use the [Amazon Kinesis Client Library for Python][kclpy] to write their data processing applications.
 
 ### Release 1.1 (June 30, 2014)
 * **Checkpointing at a specific sequence number** — The IRecordProcessorCheckpointer interface now supports checkpointing at a sequence number specified by the record processor.
@@ -57,6 +65,7 @@ To make it easier for developers to write record processors in other languages,
 [kinesis-guide-begin]: http://docs.aws.amazon.com/kinesis/latest/dev/before-you-begin.html
 [kinesis-guide-create]: http://docs.aws.amazon.com/kinesis/latest/dev/step-one-create-stream.html
 [kinesis-guide-applications]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-app.html
+[kinesis-guide-monitoring-with-kcl]: http://docs.aws.amazon.com//kinesis/latest/dev/monitoring-with-kcl.html
 [kinesis-guide-kpl]: http://docs.aws.amazon.com//kinesis/latest/dev/developing-producers-with-kpl.html
 [kinesis-guide-consumer-deaggregation]: http://docs.aws.amazon.com//kinesis/latest/dev/kinesis-kpl-consumer-deaggregation.html
 [kclpy]: https://github.com/awslabs/amazon-kinesis-client-python

pom.xml

@@ -6,7 +6,7 @@
     <artifactId>amazon-kinesis-client</artifactId>
     <packaging>jar</packaging>
     <name>Amazon Kinesis Client Library for Java</name>
-    <version>1.4.0</version>
+    <version>1.5.0</version>
     <description>The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis.</description>
     <url>https://aws.amazon.com/kinesis</url>
 
@@ -29,9 +29,29 @@
     <dependencies>
         <dependency>
             <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk</artifactId>
+            <artifactId>aws-java-sdk-core</artifactId>
             <version>${aws-java-sdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-dynamodb</artifactId>
+            <version>${aws-java-sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-kinesis</artifactId>
+            <version>${aws-java-sdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-cloudwatch</artifactId>
+            <version>${aws-java-sdk.version}</version>
+        </dependency>
+        <dependency>
+	    <groupId>com.google.guava</groupId>
+	    <artifactId>guava</artifactId>
+	    <version>18.0</version>
+        </dependency>
         <dependency>
           <groupId>com.google.protobuf</groupId>
           <artifactId>protobuf-java</artifactId>

src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/KinesisClientLibConfigurator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * Licensed under the Amazon Software License (the "License").
  * You may not use this file except in compliance with the License.
@@ -67,7 +67,8 @@ public KinesisClientLibConfigurator() {
                         new AWSCredentialsProviderPropertyValueDecoder(),
                         new StringPropertyValueDecoder(),
                         new InitialPositionInStreamPropertyValueDecoder(),
-                        new ClientConfigurationPropertyValueDecoder());
+                        new ClientConfigurationPropertyValueDecoder(),
+                        new SetPropertyValueDecoder());
 
         classToDecoder = new Hashtable<Class<?>, IPropertyValueDecoder<?>>();
         for (IPropertyValueDecoder<?> getter : getters) {

src/main/java/com/amazonaws/services/kinesis/clientlibrary/config/SetPropertyValueDecoder.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Amazon Software License (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/asl/
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+package com.amazonaws.services.kinesis.clientlibrary.config;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Provide {@link Set} property value. Note that since parameterized value cannot be figured out during compile time
+ * for setter methods, only {@code Set} of {@code String}s are supported as property value decode.
+ */
+@SuppressWarnings("rawtypes")
+class SetPropertyValueDecoder implements IPropertyValueDecoder<Set> {
+
+    /**
+     * Delimiter for the list provided as string.
+     */
+    private static final String LIST_DELIMITER = ",";
+
+    /**
+     * Package constructor for factory use only.
+     */
+    SetPropertyValueDecoder() {
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public Set decodeValue(String propertyValue) {
+        String[] values = propertyValue.split(LIST_DELIMITER);
+        String value = null;
+        Set<String> decodedValue = new HashSet<>();
+        for (int i = 0; i < values.length; i++) {
+            value = values[i].trim();
+            if (!value.isEmpty()) {
+                decodedValue.add(value);
+            }
+        }
+        return decodedValue;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public List<Class<Set>> getSupportedTypes() {
+        return Arrays.asList(Set.class);
+    }
+
+}

src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/InitializeTask.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * Licensed under the Amazon Software License (the "License").
  * You may not use this file except in compliance with the License.
@@ -21,13 +21,18 @@
 import com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor;
 import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
 import com.amazonaws.services.kinesis.clientlibrary.types.InitializationInput;
+import com.amazonaws.services.kinesis.metrics.impl.MetricsHelper;
+import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel;
 
 /**
  * Task for initializing shard position and invoking the RecordProcessor initialize() API.
  */
 class InitializeTask implements ITask {
 
     private static final Log LOG = LogFactory.getLog(InitializeTask.class);
+
+    private static final String RECORD_PROCESSOR_INITIALIZE_METRIC = "RecordProcessor.initialize";
+
     private final ShardInfo shardInfo;
     private final IRecordProcessor recordProcessor;
     private final KinesisDataFetcher dataFetcher;
@@ -72,16 +77,21 @@ public TaskResult call() {
             dataFetcher.initialize(initialCheckpoint.getSequenceNumber());
             recordProcessorCheckpointer.setLargestPermittedCheckpointValue(initialCheckpoint);
             recordProcessorCheckpointer.setInitialCheckpointValue(initialCheckpoint);
+
+            LOG.debug("Calling the record processor initialize().");
+            final InitializationInput initializationInput = new InitializationInput()
+                .withShardId(shardInfo.getShardId())
+                .withExtendedSequenceNumber(initialCheckpoint);
+            final long recordProcessorStartTimeMillis = System.currentTimeMillis();
             try {
-                LOG.debug("Calling the record processor initialize().");
-                final InitializationInput initializationInput = new InitializationInput()
-                    .withShardId(shardInfo.getShardId())
-                    .withExtendedSequenceNumber(initialCheckpoint);
                 recordProcessor.initialize(initializationInput);
                 LOG.debug("Record processor initialize() completed.");
             } catch (Exception e) {
                 applicationException = true;
                 throw e;
+            } finally {
+                MetricsHelper.addLatency(RECORD_PROCESSOR_INITIALIZE_METRIC, recordProcessorStartTimeMillis,
+                        MetricsLevel.SUMMARY);
             }
 
             return new TaskResult(null);