diff options
Diffstat (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java')
-rw-r--r-- | exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java | 92 |
1 files changed, 68 insertions, 24 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java index 55fc13d8d..95917cb6e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java @@ -20,31 +20,43 @@ package org.apache.drill.exec.store.httpd; import org.apache.drill.shaded.guava.com.google.common.base.Charsets; import org.apache.drill.shaded.guava.com.google.common.collect.Maps; import io.netty.buffer.DrillBuf; + import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; + import nl.basjes.parse.core.Casts; import nl.basjes.parse.core.Parser; import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; import org.apache.drill.exec.vector.complex.writer.BigIntWriter; import org.apache.drill.exec.vector.complex.writer.Float8Writer; import org.apache.drill.exec.vector.complex.writer.VarCharWriter; +import org.apache.drill.exec.vector.complex.writer.TimeStampWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.SimpleDateFormat; +import java.util.Date; + public class HttpdLogRecord { private static final Logger LOG = LoggerFactory.getLogger(HttpdLogRecord.class); private final Map<String, VarCharWriter> strings = Maps.newHashMap(); private final Map<String, BigIntWriter> longs = Maps.newHashMap(); private final Map<String, Float8Writer> doubles = Maps.newHashMap(); + private final Map<String, TimeStampWriter> times = new HashMap<>(); private final Map<String, MapWriter> wildcards = Maps.newHashMap(); private final Map<String, String> cleanExtensions = Maps.newHashMap(); private final Map<String, MapWriter> startedWildcards = Maps.newHashMap(); private final Map<String, MapWriter> wildcardWriters = Maps.newHashMap(); + private final SimpleDateFormat dateFormatter; private DrillBuf managedBuffer; + private String timeFormat; - public HttpdLogRecord(final DrillBuf managedBuffer) { + public HttpdLogRecord(final DrillBuf managedBuffer, final String timeFormat) { this.managedBuffer = managedBuffer; + this.timeFormat = timeFormat; + this.dateFormatter = new SimpleDateFormat(this.timeFormat); } /** @@ -66,7 +78,7 @@ public class HttpdLogRecord { return managedBuffer; } - private void writeString(final VarCharWriter writer, final String value) { + private void writeString(VarCharWriter writer, String value) { final byte[] stringBytes = value.getBytes(Charsets.UTF_8); final DrillBuf stringBuffer = buf(stringBytes.length); stringBuffer.clear(); @@ -82,14 +94,13 @@ public class HttpdLogRecord { * @param value value of field */ @SuppressWarnings("unused") - public void set(final String field, final String value) { + public void set(String field, String value) { if (value != null) { final VarCharWriter w = strings.get(field); if (w != null) { LOG.trace("Parsed field: {}, as string: {}", field, value); writeString(w, value); - } - else { + } else { LOG.warn("No 'string' writer found for field: {}", field); } } @@ -103,14 +114,13 @@ public class HttpdLogRecord { * @param value value of field */ @SuppressWarnings("unused") - public void set(final String field, final Long value) { + public void set(String field, Long value) { if (value != null) { final BigIntWriter w = longs.get(field); if (w != null) { LOG.trace("Parsed field: {}, as long: {}", field, value); w.writeBigInt(value); - } - else { + } else { LOG.warn("No 'long' writer found for field: {}", field); } } @@ -118,20 +128,48 @@ public class HttpdLogRecord { /** * This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get + * called when the value of a log field is a timesstamp data type. + * + * @param field name of field + * @param value value of field + */ + @SuppressWarnings("unused") + public void setTimestamp(String field, String value) { + if (value != null) { + //Convert the date string into a long + long ts = 0; + try { + Date d = this.dateFormatter.parse(value); + ts = d.getTime(); + } catch (Exception e) { + //If the date formatter does not successfully create a date, the timestamp will fall back to zero + //Do not throw exception + } + final TimeStampWriter tw = times.get(field); + if (tw != null) { + LOG.trace("Parsed field: {}, as time: {}", field, value); + tw.writeTimeStamp(ts); + } else { + LOG.warn("No 'timestamp' writer found for field: {}", field); + } + } + } + + /** + * This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get * called when the value of a log field is a Double data type. * * @param field name of field * @param value value of field */ @SuppressWarnings("unused") - public void set(final String field, final Double value) { + public void set(String field, Double value) { if (value != null) { final Float8Writer w = doubles.get(field); if (w != null) { LOG.trace("Parsed field: {}, as double: {}", field, value); w.writeFloat8(value); - } - else { + } else { LOG.warn("No 'double' writer found for field: {}", field); } } @@ -146,7 +184,7 @@ public class HttpdLogRecord { * @param value value of field */ @SuppressWarnings("unused") - public void setWildcard(final String field, final String value) { + public void setWildcard(String field, String value) { if (value != null) { final MapWriter mapWriter = getWildcardWriter(field); LOG.trace("Parsed wildcard field: {}, as string: {}", field, value); @@ -164,7 +202,7 @@ public class HttpdLogRecord { * @param value value of field */ @SuppressWarnings("unused") - public void setWildcard(final String field, final Long value) { + public void setWildcard(String field, Long value) { if (value != null) { final MapWriter mapWriter = getWildcardWriter(field); LOG.trace("Parsed wildcard field: {}, as long: {}", field, value); @@ -182,7 +220,7 @@ public class HttpdLogRecord { * @param value value of field */ @SuppressWarnings("unused") - public void setWildcard(final String field, final Double value) { + public void setWildcard(String field, Double value) { if (value != null) { final MapWriter mapWriter = getWildcardWriter(field); LOG.trace("Parsed wildcard field: {}, as double: {}", field, value); @@ -199,7 +237,7 @@ public class HttpdLogRecord { * @param field like HTTP.URI:request.firstline.uri.query.old where 'old' is one of many different parameter names. * @return the writer to be used for this field. */ - private MapWriter getWildcardWriter(final String field) { + private MapWriter getWildcardWriter(String field) { MapWriter writer = startedWildcards.get(field); if (writer == null) { for (Map.Entry<String, MapWriter> entry : wildcards.entrySet()) { @@ -212,7 +250,7 @@ public class HttpdLogRecord { * unsafe characters in it. */ if (!cleanExtensions.containsKey(field)) { - final String extension = field.substring(root.length() + 1, field.length()); + final String extension = field.substring(root.length() + 1); final String cleanExtension = HttpdParser.drillFormattedFieldName(extension); cleanExtensions.put(field, cleanExtension); LOG.debug("Added extension: field='{}' with cleanExtension='{}'", field, cleanExtension); @@ -255,6 +293,10 @@ public class HttpdLogRecord { return doubles; } + public Map<String, TimeStampWriter> getTimes() { + return times; + } + /** * This record will be used with a single parser. For each field that is to be parsed a setter will be called. It * registers a setter method for each field being parsed. It also builds the data writers to hold the data beings @@ -281,21 +323,23 @@ public class HttpdLogRecord { parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Double.class), parserFieldName); parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Long.class), parserFieldName); wildcards.put(cleanName, mapWriter.map(drillFieldName)); - } - else if (type.contains(Casts.DOUBLE)) { + } else if (type.contains(Casts.DOUBLE)) { LOG.debug("Adding DOUBLE parse target: {}, with field name: {}", parserFieldName, drillFieldName); parser.addParseTarget(this.getClass().getMethod("set", String.class, Double.class), parserFieldName); doubles.put(parserFieldName, mapWriter.float8(drillFieldName)); - } - else if (type.contains(Casts.LONG)) { + } else if (type.contains(Casts.LONG)) { LOG.debug("Adding LONG parse target: {}, with field name: {}", parserFieldName, drillFieldName); parser.addParseTarget(this.getClass().getMethod("set", String.class, Long.class), parserFieldName); longs.put(parserFieldName, mapWriter.bigInt(drillFieldName)); - } - else { + } else { LOG.debug("Adding STRING parse target: {}, with field name: {}", parserFieldName, drillFieldName); - parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName); - strings.put(parserFieldName, mapWriter.varChar(drillFieldName)); + if (parserFieldName.startsWith("TIME.STAMP:")) { + parser.addParseTarget(this.getClass().getMethod("setTimestamp", String.class, String.class), parserFieldName); + times.put(parserFieldName, mapWriter.timeStamp(drillFieldName)); + } else { + parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName); + strings.put(parserFieldName, mapWriter.varChar(drillFieldName)); + } } } }
\ No newline at end of file |