Improve the usability of the base docker

Dockerfile

@@ -19,3 +19,7 @@ RUN ln -s /usr/local/apache-kafka/kafka_2.11-0.9.0.1/config /config
 
 ADD kafka-bin.py /app/bin/
 ADD kafka-init /app/bin/
+ADD kafka-startup.json /app/config/
+ADD kafka-init.json /app/config/
+
+CMD ["bash", "-c", "/app/bin/kafka-init"]

README.md

@@ -16,6 +16,51 @@ To start the Kafka Docker image:
 
     docker run -i -t bde2020/docker-kafka /bin/bash
 
+### Configure Kafka
+To configure kafka we require to be provided with 2 JSON files; kafka-startup.json and kafka-init.json. Both files are expected to contain a JSON array of objects where each object represents a shell command to be executed. The kafka-startup.json should contain (at least) the command to start kafka within the docker. 
+
+There is a python script that will pick up these files when the container starts and convert the JSON objects to shell commands and execute them.
+
+As an example of the kafka-startup.json file:
+```
+[
+ {
+   "sh":"/app/bin/kafka-server-start.sh",
+   "./config/server.properties":"",
+     "--override":[
+         "-Djava.net.preferIPv4Stack=true",
+         "zookeeper.connect=zookeeper:2181"
+         ]
+ }
+]
+```
+which will, by the python script, be converted to 
+```
+sh /app/bin/kafka-server-start.sh ./config/server.properties --override -Djava.nete.preferIPv4Stack=true --override zookeeper.connect=zookeeper:2181
+```
+
+An example for the kafka-init.json would be:
+```
+[
+  {
+      "sh":"/app/bin/kafka-topics.sh",
+      "--zookeeper":"zookeeper:2181",
+      "--create":"",
+      "--topic":"sampleTopic",
+      "--partitions":"\"3\"",
+      "--replication-factor":"\"1\""
+  }
+]
+
+```
+again ending up running the following shell command:
+```
+sh /app/bin/kafka-topics.sh --zookeeper zookeeper:2181 --create --topic sampleTopic --partitions: 3 --replication-factor 1
+```
+And also again there can be other commands chained to this one by adding objects to the JSON array.
+
+### Building the image
+
 To build the Kafka Docker image:
 
  ```bash

kafka-bin.py

@@ -1,9 +1,31 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
+# this python script will execute any call that the JSON file passed as an argument describes
+# the form of this JSON should be like:
+#   [
+#     {
+#       "sh": "some-shell-script.sh",
+#       "option": "single-value-option",
+#       "list-option: [
+#                "list-option-1",
+#                "list-option-2"
+#       ]
+#     },
+#     {
+#       "sh": "some-other-shell-script.sh"
+#     }
+#   ]
+#
+# this will then be converted to as many commands as the json-array holds objects
+# for the example above the result will be that the following 2 commands are run:
+#     sh some-shell-script.sh option single-value option list-option list-option-1 list-option list-option-2
+#     sh some-other-shell-script.sh
+
 import os,sys,collections
 import subprocess as sp
 import json
+import types
 
 class KafkaBin:
 
@@ -19,15 +41,24 @@ def fromJson(self, kafka_commands_file_name=None):
             dev_null = open(os.devnull, 'w')
             for command in kafka_commands:
                 command_call=[]
-                [command_call.extend([k,(os.environ[v[1:]] if v.startswith('$') else v)]) for k,v in command.items()]
-                command_call = [var for var in command_call if var]
+                for command_start, command_end in command.items():
+                    if(isinstance(command_end, list)):
+                        for option in command_end:
+                            if(len(option) > 0):
+                                command_call.append(str(command_start))
+                                command_call.append(str(option))
+                    else:
+                        command_call.append(str(command_start))
+                        if(len(command_end) > 0):
+                            command_call.append(str(command_end))
+                print("[!] command call about to be executed: " + str(command_call))
                 if command_call[0] == "sh":
                     exit_code = sp.call(command_call, stderr=dev_null,close_fds=True)
                 else:
                     raise Exception("json construct must start with 'sh' and the path to a kafka binary");
-                
+
         pass;
-    
+
 if __name__ == '__main__':
     if(len(sys.argv)==2):
         kafkaBin = KafkaBin();

kafka-init

@@ -5,9 +5,9 @@
 KAFKA_STARTUP="kafka-startup.json"
 KAFKA_INIT="kafka-init.json"
 if [ -f "/config/$KAFKA_STARTUP" ]; then
-  /app/bin/kafka-bin.py "/config/$KAFKA_STARTUP"
-  if [ -f "/config/$KAFKA_INIT" ]; then
-    /app/bin/kafka-bin.py "/config/$KAFKA_INIT"
-  fi
+    if [ -f "/config/$KAFKA_INIT" ]; then
+        /app/bin/kafka-bin.py "/kafka-config/$KAFKA_INIT" &
+    fi
+    /app/bin/kafka-bin.py "/kafka-config/$KAFKA_STARTUP"
 fi
 /finish-step.sh

kafka-init.json

@@ -0,0 +1,10 @@
+[
+  {
+      "sh":"/app/bin/kafka-topics.sh",
+      "--zookeeper":"zookeeper:2181",
+      "--create":"",
+      "--topic":"sampleTopic",
+      "--partitions":"\"3\"",
+      "--replication-factor":"\"1\""
+  }
+]

kafka-startup.json

@@ -0,0 +1,11 @@
+[
+ {
+   "sh":"/app/bin/kafka-server-start.sh",
+   "./config/server.properties":"",
+     "--override":[
+         "-Djava.net.preferIPv4Stack=true",
+         "zookeeper.connect=zookeeper:2181",
+         "listeners=PLAINTEXT://0.0.0.0:9092,SSL://0.0.0.0:9091"
+         ]
+ }
+]