Merge pull request #1709 from oleewere/AMBARI-24261-branch-2.7

[Ambari 24261] Logfeeder fail to start due to missing conf file post logsearch upgrade

7 files changed, 128 insertions, 9 deletions

diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
index d50dbd1a4a..10dffa80d2 100644
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/deb/logfeeder/postinst
@@ -23,7 +23,7 @@ LOGFEEDER_CONF_SOURCE="/usr/lib/ambari-logsearch-logfeeder/conf"
 ln -s $LOGFEEDER_SCRIPT_SOURCE $LOGFEEDER_SCRIPT_LINK_NAME
 #ln -s $LOGFEEDER_CONF_SOURCE $LOGFEEDER_CONF_LINK
 
-# handle old checkpoint & keys folder
+# handle old keys folder & custom jsons
 
 LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old"
 
@@ -31,7 +31,12 @@ if [ -d "$LOGFEEDER_CONF_BACKUP" ]; then
   if [ -d "$LOGFEEDER_CONF_BACKUP/keys" ]; then
     cp -r $LOGFEEDER_CONF_BACKUP/keys $LOGFEEDER_CONF_SOURCE
   fi
-  if [ -d "$LOGFEEDER_CONF_BACKUP/checkpoints" ]; then
-    cp -r $LOGFEEDER_CONF_BACKUP/checkpoints $LOGFEEDER_CONF_SOURCE
+
+  custom_jsons=(`find $LOGFEEDER_CONF_BACKUP -name "*.json" ! -name 'input*.json' ! -name 'global.config.json' ! -name 'output.config.json'`)
+  if [ ! -z "$custom_jsons" ]; then
+    for custom_json_file in "${custom_jsons[@]}"
+    do :
+      cp -r $custom_json_file "$LOGFEEDER_CONF_SOURCE/"
+    done
   fi
 fi
\ No newline at end of file
diff --git a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh
index 2db0d52aa5..ce4b611ca3 100755
--- a/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh
+++ b/ambari-logsearch/ambari-logsearch-assembly/src/main/package/rpm/logfeeder/postinstall.sh
@@ -26,7 +26,7 @@ mkdir -p $LOGFEEDER_ETC_FOLDER
 ln -s $LOGFEEDER_SCRIPT_SOURCE $LOGFEEDER_SCRIPT_LINK_NAME
 #ln -s $LOGFEEDER_CONF_SOURCE $LOGFEEDER_CONF_LINK
 
-# handle old checkpoint & keys folder
+# handle old keys folder & custom jsons
 
 LOGFEEDER_CONF_BACKUP="/usr/lib/ambari-logsearch-logfeeder/conf-old"
 
@@ -34,7 +34,12 @@ if [ -d "$LOGFEEDER_CONF_BACKUP" ]; then
   if [ -d "$LOGFEEDER_CONF_BACKUP/keys" ]; then
     cp -r $LOGFEEDER_CONF_BACKUP/keys $LOGFEEDER_CONF_SOURCE
   fi
-  if [ -d "$LOGFEEDER_CONF_BACKUP/checkpoints" ]; then
-    cp -r $LOGFEEDER_CONF_BACKUP/checkpoints $LOGFEEDER_CONF_SOURCE
+
+  custom_jsons=(`find $LOGFEEDER_CONF_BACKUP -name "*.json" ! -name 'input*.json' ! -name 'global.config.json' ! -name 'output.config.json'`)
+  if [ ! -z "$custom_jsons" ]; then
+    for custom_json_file in "${custom_jsons[@]}"
+    do :
+      cp -r $custom_json_file "$LOGFEEDER_CONF_SOURCE/"
+    done
   fi
 fi
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-kafka-output-config.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-kafka-output-config.xml
new file mode 100644
index 0000000000..1f711097c9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-kafka-output-config.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>logfeeder_kafka_output_enabled</name>
+    <value>false</value>
+    <description>Enable Kafka output for Log Feeder</description>
+    <display-name>Log Feeder Kafka output enabled</display-name>
+    <value-attributes>
+      <type>boolean</type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>logfeeder_kafka_broker_list</name>
+    <value>localhost:6667</value>
+    <display-name>Log Feeder Kafka broker list</display-name>
+    <description>Broker list for Kafka output (define the right value, as it can be external, also you can add PLAINTEXT or PLAINTEXTSASL protocol as well to the addresses)</description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>logfeeder_kafka_topic</name>
+    <value>log-streaming</value>
+    <display-name>Log Feeder Kafka topic</display-name>
+    <description>Kafka topic which is the destination of Log Feeder outputs</description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>content</name>
+    <display-name>Log Feeder Kafka output config</display-name>
+    <description>Kafka output configuration for Log Feeder (data shipping)</description>
+    <value/>
+    <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+      <property-file-name>kafka-output.json.j2</property-file-name>
+      <property-file-type>text</property-file-type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
index 074717abe5..1b7643f2a1 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-properties.xml
@@ -35,7 +35,7 @@
     <value-attributes>
       <type>directory</type>
     </value-attributes>
-    <on-ambari-upgrade add="true"/>
+    <on-ambari-upgrade add="true" update="true"/>
   </property>
   <property>
     <name>logfeeder.metrics.collector.hosts</name>
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index a492175bee..6f30144ec7 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -340,8 +340,18 @@ logfeeder_env_jceks_file = os.path.join(logsearch_logfeeder_conf, 'logfeeder.jce
 
 logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content']
 logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content']
+logfeeder_kafka_output_config_content = config['configurations']['logfeeder-kafka-output-config']['content']
 
-default_config_files = ','.join(['output.config.json','global.config.json'])
+logfeeder_kafka_output_enabled = False
+if 'logfeeder-kafka-output-config' in config['configurations']:
+  logfeeder_kafka_output_enabled = default('/configurations/logfeeder-kafka-output-config/logfeeder_kafka_output_enabled', False)
+  logfeeder_kafka_broker_list = default('/configurations/logfeeder-kafka-output-config/logfeeder_kafka_broker_list', 'localhost:6667')
+  logfeeder_kafka_topic = default('/configurations/logfeeder-kafka-output-config/logfeeder_kafka_topic', 'log-streaming')
+
+if logfeeder_kafka_output_enabled:
+  default_config_files = ','.join(['output.config.json','global.config.json', 'kafka-output.json'])
+else:
+  default_config_files = ','.join(['output.config.json','global.config.json'])
 
 logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns']
 if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip():
@@ -395,7 +405,6 @@ logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder'
 
 logfeeder_use_ssl = logsearch_solr_ssl_enabled or metrics_collector_protocol == 'https'
 
-
 logsearch_acls = ''
 if 'infra-solr-env' in config['configurations'] and security_enabled and not logsearch_use_external_solr:
   acl_infra_solr_principal = get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal'])
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index 76ec5c56d5..067da8c23a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -113,6 +113,11 @@ def setup_logfeeder():
        content=InlineTemplate(params.logfeeder_output_config_content),
        encoding="utf-8"
        )
+  if params.logfeeder_kafka_output_enabled:
+    File(format("{logsearch_logfeeder_conf}/kafka-output.json"),
+         content=InlineTemplate(params.logfeeder_kafka_output_config_content),
+         encoding="utf-8"
+         )
 
   if params.logfeeder_system_log_enabled:
     File(format("{logsearch_logfeeder_conf}/input.config-system_messages.json"),
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/kafka-output.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/kafka-output.json.j2
new file mode 100644
index 0000000000..ceb1f94bdf
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/kafka-output.json.j2
@@ -0,0 +1,34 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+ {
+   "output": [
+     {
+       "is_enabled": "{{ logfeeder_kafka_output_enabled | lower }}",
+       "destination": "kafka",
+       "broker_list": "{{ logfeeder_kafka_broker_list }}",
+       "topic": "{{ logfeeder_kafka_topic }}",
+       "conditions": {
+         "fields": {
+           "rowtype": [
+             "service"
+           ]
+         }
+       }
+     }
+   ]
+ }
\ No newline at end of file