JAVAFICATION: Move env/secret variable substitution to java

This is a partial port that required writing our own version of ruby's gsub. Java has `String#replaceAll`, but that doesn't let you pass in a function to do the replacement.
elastic · May 19, 2018 · 068a515 · 068a515
1 parent a360dbb
commit 068a515
Show file tree

Hide file tree

Showing 8 changed files with 244 additions and 76 deletions.
diff --git a/logstash-core/lib/logstash/util/substitution_variables.rb b/logstash-core/lib/logstash/util/substitution_variables.rb
@@ -31,26 +31,11 @@ def deep_replace(value)
   def replace_placeholders(value)
     return value unless value.is_a?(String)
 
-    value.gsub(SUBSTITUTION_PLACEHOLDER_REGEX) do |placeholder|
-      # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to
-      # the call, so this should be thread-safe.
-      #
-      # [1] http://ruby-doc.org/core-2.1.1/Regexp.html#method-c-last_match
-      name = Regexp.last_match(:name)
-      default = Regexp.last_match(:default)
-      logger.debug("Replacing `#{placeholder}` with actual value")
-
-      #check the secret store if it exists
-      secret_store = LogStash::Util::SecretStore.get_if_exists
-      replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(LogStash::Util::SecretStore.get_store_id(name))
-      #check the environment
-      replacement = ENV.fetch(name, default) if replacement.nil?
-      if replacement.nil?
-        raise LogStash::ConfigurationError, "Cannot evaluate `#{placeholder}`. Replacement variable `#{name}` is not defined in a Logstash secret store " +
-            "or as an Environment entry and there is no default value given."
-      end
-      replacement.to_s
-    end
+    org.logstash.common.SubstitutionVariables.replacePlaceholders(
+        value, ENV, LogStash::Util::SecretStore.get_if_exists
+    );
+  rescue org.logstash.common.SubstitutionVariables::MissingSubstitutionVariableError => e
+    raise ::LogStash::ConfigurationError, e.getMessage
   end # def replace_placeholders
 
 end
diff --git a/logstash-core/src/main/java/org/logstash/common/SubstitutionVariables.java b/logstash-core/src/main/java/org/logstash/common/SubstitutionVariables.java
@@ -0,0 +1,42 @@
+package org.logstash.common;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.logstash.secret.SecretIdentifier;
+import org.logstash.secret.store.SecretStore;
+import org.logstash.secret.store.SecretStoreFactory;
+
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public class SubstitutionVariables {
+    private static final Logger logger = LogManager.getLogger(SubstitutionVariables.class);
+
+    private static Pattern SUBSTITUTION_PLACEHOLDER_REGEX = Pattern.compile("\\$\\{(?<name>[a-zA-Z_.][a-zA-Z0-9_.]*)(:(?<default>[^}]*))?\\}");
+
+    static String replacePlaceholders(final String input,
+                                      final Map<String, String> environmentVariables,
+                                      final SecretStore secretStore) {
+
+        return Util.gsub(input, SUBSTITUTION_PLACEHOLDER_REGEX, (matchResult) -> {
+            String name = matchResult.group(1);
+            String defaultValue = matchResult.group(3);
+
+            if (secretStore != null) {
+                    byte[] secretValue = secretStore.retrieveSecret(new SecretIdentifier(name));
+                    if (secretValue != null) return new String(secretValue);
+                }
+
+                String envValue = environmentVariables.getOrDefault(name, defaultValue);
+                if (envValue != null) return envValue;
+
+                String errMsg = String.format(
+                        "Cannot evaluate `%s`. Replacement variable `%s` is not defined in a Logstash secret store " +
+                                "or as an Environment entry and there is no default value given.",
+                        matchResult.group(0), name);
+                throw new MissingSubstitutionVariableError();
+        });
+    }
+
+    static class MissingSubstitutionVariableError extends RuntimeException {}
+}
diff --git a/logstash-core/src/main/java/org/logstash/common/Util.java b/logstash-core/src/main/java/org/logstash/common/Util.java
@@ -3,6 +3,10 @@
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.function.Function;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Created by andrewvc on 12/23/16.
@@ -35,4 +39,43 @@ public static String bytesToHexString(byte[] bytes) {
 
         return hexString.toString();
     }
+
+    /**
+     * Replace the given regex with a new value based on the given function
+     * @param input The string to search
+     * @param pattern regex pattern string
+     * @param matchSubstituter function that does the replacement based on the match
+     * @return new string, with substitutions
+     */
+    public static String gsub(final String input, final String pattern, Function<MatchResult, String> matchSubstituter) {
+        return gsub(input, Pattern.compile(pattern), matchSubstituter);
+    }
+
+    /**
+     * Replace the given regex with a new value based on the given function
+     * @param input The string to search
+     * @param pattern Compiled regex pattern
+     * @param matchSubstituter function that does the replacement based on the match
+     * @return new string, with substitutions
+     */
+    public static String gsub(final String input, final Pattern pattern, Function<MatchResult, String> matchSubstituter) {
+        final StringBuilder output = new StringBuilder();
+        final Matcher matcher = pattern.matcher(input);
+
+        while (matcher.find()) {
+            // Add the non-matched text preceding the match to the output
+            output.append(input, matcher.regionStart(), matcher.start());
+
+            // Add the substituted match to the output
+            output.append(matchSubstituter.apply(matcher.toMatchResult()));
+
+            // Move the matched region to after the match
+            matcher.region(matcher.end(), input.length());
+        }
+
+        // slurp remaining into output
+        output.append(input, matcher.regionStart(), input.length());
+
+        return output.toString();
+    }
 }
diff --git a/logstash-core/src/test/java/org/logstash/common/SubstitutionVariablesTest.java b/logstash-core/src/test/java/org/logstash/common/SubstitutionVariablesTest.java
@@ -0,0 +1,51 @@
+package org.logstash.common;
+
+import org.junit.Test;
+import org.logstash.secret.MemoryStore;
+import org.logstash.secret.SecretIdentifier;
+import org.logstash.secret.store.SecretStore;
+
+import java.util.Collections;
+
+import static org.junit.Assert.*;
+
+public class SubstitutionVariablesTest {
+    @Test
+    public void substituteDefaultTest() {
+        assertEquals(
+            "Some bar Text",
+            SubstitutionVariables.replacePlaceholders("Some ${foo:bar} Text", Collections.emptyMap(), null)
+        );
+    }
+
+    @Test
+    public void substituteEnvMatchTest() {
+        assertEquals(
+            "Some env Text",
+            SubstitutionVariables.replacePlaceholders(
+                "Some ${foo:bar} Text",
+                Collections.singletonMap("foo", "env"),
+                null
+            )
+        );
+    }
+
+    @Test
+    public void substituteSecretMatchTest() {
+        SecretStore secretStore = new MemoryStore();
+        SecretIdentifier identifier = new SecretIdentifier("foo");
+        String secretValue = "SuperSekret";
+        secretStore.persistSecret(identifier, secretValue.getBytes());
+
+        assertEquals(
+            "Some " + secretValue + " Text",
+            SubstitutionVariables.replacePlaceholders(
+                "Some ${foo:bar} Text",
+                // Tests precedence over the env as well
+                Collections.singletonMap("foo", "env"),
+                secretStore
+            )
+        );
+    }
+
+}
diff --git a/logstash-core/src/test/java/org/logstash/common/UtilTest.java b/logstash-core/src/test/java/org/logstash/common/UtilTest.java
@@ -0,0 +1,37 @@
+package org.logstash.common;
+
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class UtilTest {
+
+    @Test
+    public void gsubSimple() {
+        assertEquals(
+            "fooREPLACEbarREPLACEbaz",
+            Util.gsub(
+                    "fooXbarXbaz",
+                    "X",
+                    (mRes) -> "REPLACE"
+            )
+        );
+    }
+
+    @Test
+    public void gsubGroups() {
+        assertEquals(
+            "fooYbarZbaz",
+            Util.gsub(
+                    "foo${Y}bar${Z}baz",
+                    "\\$\\{(.)\\}", (mRes) -> mRes.group(1)));
+    }
+
+    @Test
+    public void gsubNoMatch() {
+        assertEquals(
+            "foobarbaz",
+            Util.gsub("foobarbaz", "XXX", (mRes) -> "")
+        );
+    }
+}
diff --git a/logstash-core/src/test/java/org/logstash/secret/MemoryStore.java b/logstash-core/src/test/java/org/logstash/secret/MemoryStore.java
@@ -0,0 +1,64 @@
+package org.logstash.secret;
+
+import org.logstash.secret.store.SecretStore;
+import org.logstash.secret.store.SecureConfig;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.logstash.secret.store.SecretStoreFactory.LOGSTASH_MARKER;
+
+/**
+ * Valid alternate implementation of secret store
+ */
+public class MemoryStore implements SecretStore {
+
+    Map<SecretIdentifier, ByteBuffer> secrets = new HashMap(1);
+
+    public MemoryStore() {
+        persistSecret(LOGSTASH_MARKER, LOGSTASH_MARKER.getKey().getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public SecretStore create(SecureConfig secureConfig) {
+        return this;
+    }
+
+    @Override
+    public void delete(SecureConfig secureConfig) {
+        secrets.clear();
+    }
+
+    @Override
+    public SecretStore load(SecureConfig secureConfig) {
+        return this;
+    }
+
+    @Override
+    public boolean exists(SecureConfig secureConfig) {
+        return true;
+    }
+
+    @Override
+    public Collection<SecretIdentifier> list() {
+        return secrets.keySet();
+    }
+
+    @Override
+    public void persistSecret(SecretIdentifier id, byte[] secret) {
+        secrets.put(id, ByteBuffer.wrap(secret));
+    }
+
+    @Override
+    public void purgeSecret(SecretIdentifier id) {
+        secrets.remove(id);
+    }
+
+    @Override
+    public byte[] retrieveSecret(SecretIdentifier id) {
+        return secrets.get(id).array();
+    }
+}
diff --git a/logstash-core/src/test/java/org/logstash/secret/store/SecretStoreFactoryTest.java b/logstash-core/src/test/java/org/logstash/secret/store/SecretStoreFactoryTest.java
@@ -4,13 +4,12 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
+import org.logstash.secret.MemoryStore;
 import org.logstash.secret.SecretIdentifier;
 import org.logstash.secret.store.backend.JavaKeyStore;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -36,7 +35,7 @@ public class SecretStoreFactoryTest {
     @Test
     public void testAlternativeImplementation() {
         SecureConfig secureConfig = new SecureConfig();
-        secureConfig.add("keystore.classname", "org.logstash.secret.store.SecretStoreFactoryTest$MemoryStore".toCharArray());
+        secureConfig.add("keystore.classname", "org.logstash.secret.MemoryStore".toCharArray());
         SecretStore secretStore = secretStoreFactory.load(secureConfig);
         assertThat(secretStore).isInstanceOf(MemoryStore.class);
         validateMarker(secretStore);
@@ -129,56 +128,4 @@ private void validateMarker(SecretStore secretStore) {
         assertThat(new String(marker, StandardCharsets.UTF_8)).isEqualTo(LOGSTASH_MARKER.getKey());
     }
 
-    /**
-     * Valid alternate implementation
-     */
-    static class MemoryStore implements SecretStore {
-
-        Map<SecretIdentifier, ByteBuffer> secrets = new HashMap(1);
-
-        public MemoryStore() {
-            persistSecret(LOGSTASH_MARKER, LOGSTASH_MARKER.getKey().getBytes(StandardCharsets.UTF_8));
-        }
-
-        @Override
-        public SecretStore create(SecureConfig secureConfig) {
-            return this;
-        }
-
-        @Override
-        public void delete(SecureConfig secureConfig) {
-            secrets.clear();
-        }
-
-        @Override
-        public SecretStore load(SecureConfig secureConfig) {
-            return this;
-        }
-
-        @Override
-        public boolean exists(SecureConfig secureConfig) {
-            return true;
-        }
-
-        @Override
-        public Collection<SecretIdentifier> list() {
-            return secrets.keySet();
-        }
-
-        @Override
-        public void persistSecret(SecretIdentifier id, byte[] secret) {
-            secrets.put(id, ByteBuffer.wrap(secret));
-        }
-
-        @Override
-        public void purgeSecret(SecretIdentifier id) {
-            secrets.remove(id);
-        }
-
-        @Override
-        public byte[] retrieveSecret(SecretIdentifier id) {
-            return secrets.get(id).array();
-        }
-    }
-
 }
diff --git a/logstash-core/src/test/java/org/logstash/secret/store/backend/JavaKeyStoreTest.java b/logstash-core/src/test/java/org/logstash/secret/store/backend/JavaKeyStoreTest.java
@@ -33,7 +33,6 @@
 import static java.nio.file.attribute.PosixFilePermission.*;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Fail.fail;
-import static org.hamcrest.CoreMatchers.isA;
 import static org.logstash.secret.store.SecretStoreFactory.LOGSTASH_MARKER;
 
 /**