aboutsummaryrefslogtreecommitdiff
path: root/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
diff options
context:
space:
mode:
Diffstat (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java')
-rw-r--r--exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java92
1 files changed, 68 insertions, 24 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
index 55fc13d8d..95917cb6e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
@@ -20,31 +20,43 @@ package org.apache.drill.exec.store.httpd;
import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import io.netty.buffer.DrillBuf;
+
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.Map;
+
import nl.basjes.parse.core.Casts;
import nl.basjes.parse.core.Parser;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BigIntWriter;
import org.apache.drill.exec.vector.complex.writer.Float8Writer;
import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
+import org.apache.drill.exec.vector.complex.writer.TimeStampWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
public class HttpdLogRecord {
private static final Logger LOG = LoggerFactory.getLogger(HttpdLogRecord.class);
private final Map<String, VarCharWriter> strings = Maps.newHashMap();
private final Map<String, BigIntWriter> longs = Maps.newHashMap();
private final Map<String, Float8Writer> doubles = Maps.newHashMap();
+ private final Map<String, TimeStampWriter> times = new HashMap<>();
private final Map<String, MapWriter> wildcards = Maps.newHashMap();
private final Map<String, String> cleanExtensions = Maps.newHashMap();
private final Map<String, MapWriter> startedWildcards = Maps.newHashMap();
private final Map<String, MapWriter> wildcardWriters = Maps.newHashMap();
+ private final SimpleDateFormat dateFormatter;
private DrillBuf managedBuffer;
+ private String timeFormat;
- public HttpdLogRecord(final DrillBuf managedBuffer) {
+ public HttpdLogRecord(final DrillBuf managedBuffer, final String timeFormat) {
this.managedBuffer = managedBuffer;
+ this.timeFormat = timeFormat;
+ this.dateFormatter = new SimpleDateFormat(this.timeFormat);
}
/**
@@ -66,7 +78,7 @@ public class HttpdLogRecord {
return managedBuffer;
}
- private void writeString(final VarCharWriter writer, final String value) {
+ private void writeString(VarCharWriter writer, String value) {
final byte[] stringBytes = value.getBytes(Charsets.UTF_8);
final DrillBuf stringBuffer = buf(stringBytes.length);
stringBuffer.clear();
@@ -82,14 +94,13 @@ public class HttpdLogRecord {
* @param value value of field
*/
@SuppressWarnings("unused")
- public void set(final String field, final String value) {
+ public void set(String field, String value) {
if (value != null) {
final VarCharWriter w = strings.get(field);
if (w != null) {
LOG.trace("Parsed field: {}, as string: {}", field, value);
writeString(w, value);
- }
- else {
+ } else {
LOG.warn("No 'string' writer found for field: {}", field);
}
}
@@ -103,14 +114,13 @@ public class HttpdLogRecord {
* @param value value of field
*/
@SuppressWarnings("unused")
- public void set(final String field, final Long value) {
+ public void set(String field, Long value) {
if (value != null) {
final BigIntWriter w = longs.get(field);
if (w != null) {
LOG.trace("Parsed field: {}, as long: {}", field, value);
w.writeBigInt(value);
- }
- else {
+ } else {
LOG.warn("No 'long' writer found for field: {}", field);
}
}
@@ -118,20 +128,48 @@ public class HttpdLogRecord {
/**
* This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get
+ * called when the value of a log field is a timesstamp data type.
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void setTimestamp(String field, String value) {
+ if (value != null) {
+ //Convert the date string into a long
+ long ts = 0;
+ try {
+ Date d = this.dateFormatter.parse(value);
+ ts = d.getTime();
+ } catch (Exception e) {
+ //If the date formatter does not successfully create a date, the timestamp will fall back to zero
+ //Do not throw exception
+ }
+ final TimeStampWriter tw = times.get(field);
+ if (tw != null) {
+ LOG.trace("Parsed field: {}, as time: {}", field, value);
+ tw.writeTimeStamp(ts);
+ } else {
+ LOG.warn("No 'timestamp' writer found for field: {}", field);
+ }
+ }
+ }
+
+ /**
+ * This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get
* called when the value of a log field is a Double data type.
*
* @param field name of field
* @param value value of field
*/
@SuppressWarnings("unused")
- public void set(final String field, final Double value) {
+ public void set(String field, Double value) {
if (value != null) {
final Float8Writer w = doubles.get(field);
if (w != null) {
LOG.trace("Parsed field: {}, as double: {}", field, value);
w.writeFloat8(value);
- }
- else {
+ } else {
LOG.warn("No 'double' writer found for field: {}", field);
}
}
@@ -146,7 +184,7 @@ public class HttpdLogRecord {
* @param value value of field
*/
@SuppressWarnings("unused")
- public void setWildcard(final String field, final String value) {
+ public void setWildcard(String field, String value) {
if (value != null) {
final MapWriter mapWriter = getWildcardWriter(field);
LOG.trace("Parsed wildcard field: {}, as string: {}", field, value);
@@ -164,7 +202,7 @@ public class HttpdLogRecord {
* @param value value of field
*/
@SuppressWarnings("unused")
- public void setWildcard(final String field, final Long value) {
+ public void setWildcard(String field, Long value) {
if (value != null) {
final MapWriter mapWriter = getWildcardWriter(field);
LOG.trace("Parsed wildcard field: {}, as long: {}", field, value);
@@ -182,7 +220,7 @@ public class HttpdLogRecord {
* @param value value of field
*/
@SuppressWarnings("unused")
- public void setWildcard(final String field, final Double value) {
+ public void setWildcard(String field, Double value) {
if (value != null) {
final MapWriter mapWriter = getWildcardWriter(field);
LOG.trace("Parsed wildcard field: {}, as double: {}", field, value);
@@ -199,7 +237,7 @@ public class HttpdLogRecord {
* @param field like HTTP.URI:request.firstline.uri.query.old where 'old' is one of many different parameter names.
* @return the writer to be used for this field.
*/
- private MapWriter getWildcardWriter(final String field) {
+ private MapWriter getWildcardWriter(String field) {
MapWriter writer = startedWildcards.get(field);
if (writer == null) {
for (Map.Entry<String, MapWriter> entry : wildcards.entrySet()) {
@@ -212,7 +250,7 @@ public class HttpdLogRecord {
* unsafe characters in it.
*/
if (!cleanExtensions.containsKey(field)) {
- final String extension = field.substring(root.length() + 1, field.length());
+ final String extension = field.substring(root.length() + 1);
final String cleanExtension = HttpdParser.drillFormattedFieldName(extension);
cleanExtensions.put(field, cleanExtension);
LOG.debug("Added extension: field='{}' with cleanExtension='{}'", field, cleanExtension);
@@ -255,6 +293,10 @@ public class HttpdLogRecord {
return doubles;
}
+ public Map<String, TimeStampWriter> getTimes() {
+ return times;
+ }
+
/**
* This record will be used with a single parser. For each field that is to be parsed a setter will be called. It
* registers a setter method for each field being parsed. It also builds the data writers to hold the data beings
@@ -281,21 +323,23 @@ public class HttpdLogRecord {
parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Double.class), parserFieldName);
parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Long.class), parserFieldName);
wildcards.put(cleanName, mapWriter.map(drillFieldName));
- }
- else if (type.contains(Casts.DOUBLE)) {
+ } else if (type.contains(Casts.DOUBLE)) {
LOG.debug("Adding DOUBLE parse target: {}, with field name: {}", parserFieldName, drillFieldName);
parser.addParseTarget(this.getClass().getMethod("set", String.class, Double.class), parserFieldName);
doubles.put(parserFieldName, mapWriter.float8(drillFieldName));
- }
- else if (type.contains(Casts.LONG)) {
+ } else if (type.contains(Casts.LONG)) {
LOG.debug("Adding LONG parse target: {}, with field name: {}", parserFieldName, drillFieldName);
parser.addParseTarget(this.getClass().getMethod("set", String.class, Long.class), parserFieldName);
longs.put(parserFieldName, mapWriter.bigInt(drillFieldName));
- }
- else {
+ } else {
LOG.debug("Adding STRING parse target: {}, with field name: {}", parserFieldName, drillFieldName);
- parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName);
- strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
+ if (parserFieldName.startsWith("TIME.STAMP:")) {
+ parser.addParseTarget(this.getClass().getMethod("setTimestamp", String.class, String.class), parserFieldName);
+ times.put(parserFieldName, mapWriter.timeStamp(drillFieldName));
+ } else {
+ parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName);
+ strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
+ }
}
}
} \ No newline at end of file