aboutsummaryrefslogtreecommitdiff
path: root/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
diff options
context:
space:
mode:
Diffstat (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java')
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java114
1 files changed, 31 insertions, 83 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
index 5a78732af..4785da131 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
@@ -18,9 +18,9 @@
package org.apache.drill.exec.store.httpd;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
-import com.fasterxml.jackson.annotation.JsonInclude;
import nl.basjes.parse.core.exceptions.DissectionFailure;
import nl.basjes.parse.core.exceptions.InvalidDissectorException;
import nl.basjes.parse.core.exceptions.MissingDissectorsException;
@@ -28,7 +28,6 @@ import nl.basjes.parse.core.exceptions.MissingDissectorsException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.FormatPluginConfig;
import org.apache.drill.common.logical.StoragePluginConfig;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.FragmentContext;
@@ -55,26 +54,23 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.LineRecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
+import java.util.Collections;
import java.util.Map;
import org.apache.drill.exec.store.RecordReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.HttpdLogFormatConfig> {
+public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatConfig> {
private static final Logger LOG = LoggerFactory.getLogger(HttpdLogFormatPlugin.class);
private static final String PLUGIN_EXTENSION = "httpd";
private static final int VECTOR_MEMORY_ALLOCATION = 4095;
public HttpdLogFormatPlugin(final String name, final DrillbitContext context, final Configuration fsConf,
- final StoragePluginConfig storageConfig, final HttpdLogFormatConfig formatConfig) {
+ final StoragePluginConfig storageConfig, final HttpdLogFormatConfig formatConfig) {
super(name, context, fsConf, storageConfig, formatConfig, true, false, true, true,
- Lists.newArrayList(PLUGIN_EXTENSION), PLUGIN_EXTENSION);
+ Collections.singletonList(PLUGIN_EXTENSION), PLUGIN_EXTENSION);
}
@Override
@@ -93,55 +89,6 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
}
/**
- * This class is a POJO to hold the configuration for the HttpdLogFormat Parser. This is automatically
- * serialized/deserialized from JSON format.
- */
- @JsonTypeName(PLUGIN_EXTENSION) @JsonInclude(JsonInclude.Include.NON_DEFAULT)
- public static class HttpdLogFormatConfig implements FormatPluginConfig {
-
- public String logFormat;
- public String timestampFormat;
-
- /**
- * @return the logFormat
- */
- public String getLogFormat() {
- return logFormat;
- }
-
- /**
- * @return the timestampFormat
- */
- public String getTimestampFormat() {
- return timestampFormat;
- }
-
- @Override
- public int hashCode() {
- int result = logFormat != null ? logFormat.hashCode() : 0;
- result = 31 * result + (timestampFormat != null ? timestampFormat.hashCode() : 0);
- return result;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- HttpdLogFormatConfig that = (HttpdLogFormatConfig) o;
-
- if (logFormat != null ? !logFormat.equals(that.logFormat) : that.logFormat != null) {
- return false;
- }
- return timestampFormat != null ? timestampFormat.equals(that.timestampFormat) : that.timestampFormat == null;
- }
- }
-
- /**
* This class performs the work for the plugin. This is where all logic goes to read records. In this case httpd logs
* are lines terminated with a new line character.
*/
@@ -169,11 +116,15 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
* @return Map with Drill field names as a key and Parser Field names as a value
*/
private Map<String, String> makeParserFields() {
- final Map<String, String> fieldMapping = Maps.newHashMap();
+ Map<String, String> fieldMapping = new HashMap<>();
for (final SchemaPath sp : getColumns()) {
- final String drillField = sp.getRootSegment().getPath();
- final String parserField = HttpdParser.parserFormattedFieldName(drillField);
- fieldMapping.put(drillField, parserField);
+ String drillField = sp.getRootSegment().getPath();
+ try {
+ String parserField = HttpdParser.parserFormattedFieldName(drillField);
+ fieldMapping.put(drillField, parserField);
+ } catch (Exception e) {
+ LOG.info("Putting field: " + drillField + " into map", e);
+ }
}
return fieldMapping;
}
@@ -187,10 +138,11 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
*/
final Map<String, String> fieldMapping = !isStarQuery() ? makeParserFields() : null;
writer = new VectorContainerWriter(output);
+
parser = new HttpdParser(writer.rootAsMap(), context.getManagedBuffer(),
- HttpdLogFormatPlugin.this.getConfig().getLogFormat(),
- HttpdLogFormatPlugin.this.getConfig().getTimestampFormat(),
- fieldMapping);
+ HttpdLogFormatPlugin.this.getConfig().getLogFormat(),
+ HttpdLogFormatPlugin.this.getConfig().getTimestampFormat(),
+ fieldMapping);
final Path path = fs.makeQualified(work.getPath());
FileSplit split = new FileSplit(path, work.getStart(), work.getLength(), new String[]{""});
@@ -200,23 +152,21 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
job.setInputFormat(inputFormat.getClass());
lineReader = (LineRecordReader) inputFormat.getRecordReader(split, job, Reporter.NULL);
lineNumber = lineReader.createKey();
- }
- catch (NoSuchMethodException | MissingDissectorsException | InvalidDissectorException e) {
+ } catch (NoSuchMethodException | MissingDissectorsException | InvalidDissectorException e) {
throw handleAndGenerate("Failure creating HttpdParser", e);
- }
- catch (IOException e) {
+ } catch (IOException e) {
throw handleAndGenerate("Failure creating HttpdRecordReader", e);
}
}
private RuntimeException handleAndGenerate(final String s, final Exception e) {
throw UserException.dataReadError(e)
- .message(s + "\n%s", e.getMessage())
- .addContext("Path", work.getPath())
- .addContext("Split Start", work.getStart())
- .addContext("Split Length", work.getLength())
- .addContext("Local Line Number", lineNumber.get())
- .build(LOG);
+ .message(s + "\n%s", e.getMessage())
+ .addContext("Path", work.getPath())
+ .addContext("Split Start", work.getStart())
+ .addContext("Split Length", work.getLength())
+ .addContext("Local Line Number", lineNumber.get())
+ .build(LOG);
}
/**
@@ -241,8 +191,7 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
writer.setValueCount(recordCount);
return recordCount;
- }
- catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
+ } catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
throw handleAndGenerate("Failure while parsing log record.", e);
}
}
@@ -253,8 +202,7 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
if (lineReader != null) {
lineReader.close();
}
- }
- catch (IOException e) {
+ } catch (IOException e) {
LOG.warn("Failure while closing Httpd reader.", e);
}
}
@@ -262,10 +210,10 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
@Override
public String toString() {
return "HttpdLogRecordReader[Path=" + work.getPath()
- + ", Start=" + work.getStart()
- + ", Length=" + work.getLength()
- + ", Line=" + lineNumber.get()
- + "]";
+ + ", Start=" + work.getStart()
+ + ", Length=" + work.getLength()
+ + ", Line=" + lineNumber.get()
+ + "]";
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-20 05:40:46 +0000