diff options
Diffstat (limited to 'exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java')
| -rw-r--r-- | exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java | 396 |
1 files changed, 276 insertions, 120 deletions
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java index ba72177d9..5d3d7c0d3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java @@ -45,112 +45,262 @@ public class HttpdParser { private final Parser<HttpdLogRecord> parser; private final HttpdLogRecord record; - public static final HashMap<String, String> LOGFIELDS = new HashMap<String, String>(); - static - { - LOGFIELDS.put("request_receive_time_weekyear__utc", "TIME_YEAR:request_receive_time_weekyear__utc"); - LOGFIELDS.put("request_referer_ref", "HTTP_REF:request_referer_ref"); - LOGFIELDS.put("request_referer_protocol", "HTTP_PROTOCOL:request_referer_protocol"); - LOGFIELDS.put("request_receive_time_timezone", "TIME_ZONE:request_receive_time_timezone"); - LOGFIELDS.put("connection_client_host", "IP:connection_client_host"); - LOGFIELDS.put("connection_client_ip", "IP:connection_client_ip"); - LOGFIELDS.put("connection_client_peerip", "IP:connection_client_peerip"); - LOGFIELDS.put("connection_server_ip", "IP:connection_server_ip"); - LOGFIELDS.put("request_receive_time_day", "TIME_DAY:request_receive_time_day"); - LOGFIELDS.put("request_receive_time_minute__utc", "TIME_MINUTE:request_receive_time_minute__utc"); - LOGFIELDS.put("request_referer_query_$", "STRING:request_referer_query_$"); - LOGFIELDS.put("request_receive_time_millisecond__utc", "TIME_MILLISECOND:request_receive_time_millisecond__utc"); - LOGFIELDS.put("request_firstline_uri_port", "HTTP_PORT:request_firstline_uri_port"); - LOGFIELDS.put("request_referer_userinfo", "HTTP_USERINFO:request_referer_userinfo"); - LOGFIELDS.put("request_receive_time_second__utc", "TIME_SECOND:request_receive_time_second__utc"); - LOGFIELDS.put("request_firstline_uri_protocol", "HTTP_PROTOCOL:request_firstline_uri_protocol"); - LOGFIELDS.put("request_receive_time_month", "TIME_MONTH:request_receive_time_month"); - LOGFIELDS.put("request_firstline_uri_query", "HTTP_QUERYSTRING:request_firstline_uri_query"); - LOGFIELDS.put("request_firstline_uri_path", "HTTP_PATH:request_firstline_uri_path"); - LOGFIELDS.put("request_receive_time_hour__utc", "TIME_HOUR:request_receive_time_hour__utc"); - LOGFIELDS.put("request_receive_time_monthname", "TIME_MONTHNAME:request_receive_time_monthname"); - LOGFIELDS.put("request_receive_time_year__utc", "TIME_YEAR:request_receive_time_year__utc"); - LOGFIELDS.put("request_receive_time_second", "TIME_SECOND:request_receive_time_second"); - LOGFIELDS.put("request_referer", "HTTP_URI:request_referer"); - LOGFIELDS.put("request_receive_time_monthname__utc", "TIME_MONTHNAME:request_receive_time_monthname__utc"); - LOGFIELDS.put("request_referer_path", "HTTP_PATH:request_referer_path"); - LOGFIELDS.put("request_receive_time_weekyear", "TIME_YEAR:request_receive_time_weekyear"); - LOGFIELDS.put("request_firstline_protocol", "HTTP_PROTOCOL:request_firstline_protocol"); - LOGFIELDS.put("request_referer_port", "HTTP_PORT:request_referer_port"); - LOGFIELDS.put("request_receive_time_minute", "TIME_MINUTE:request_receive_time_minute"); - LOGFIELDS.put("request_status_last", "STRING:request_status_last"); - LOGFIELDS.put("request_receive_time_hour", "TIME_HOUR:request_receive_time_hour"); - LOGFIELDS.put("request_firstline_protocol_version", "HTTP_PROTOCOL_VERSION:request_firstline_protocol_version"); - LOGFIELDS.put("request_receive_time", "TIME_STAMP:request_receive_time"); - LOGFIELDS.put("request_firstline_method", "HTTP_METHOD:request_firstline_method"); - LOGFIELDS.put("request_receive_time_epoch", "TIME_EPOCH:request_receive_time_epoch"); - LOGFIELDS.put("request_receive_time_weekofweekyear", "TIME_WEEK:request_receive_time_weekofweekyear"); - LOGFIELDS.put("request_firstline_uri_host", "HTTP_HOST:request_firstline_uri_host"); - LOGFIELDS.put("request_referer_query", "HTTP_QUERYSTRING:request_referer_query"); - LOGFIELDS.put("request_firstline_uri_userinfo", "HTTP_USERINFO:request_firstline_uri_userinfo"); - LOGFIELDS.put("response_body_bytes", "BYTES:response_body_bytes"); - LOGFIELDS.put("response_body_bytesclf", "BYTES:response_body_bytesclf"); - LOGFIELDS.put("request_referer_host", "HTTP_HOST:request_referer_host"); - LOGFIELDS.put("request_receive_time_weekofweekyear__utc", "TIME_WEEK:request_receive_time_weekofweekyear__utc"); - LOGFIELDS.put("request_firstline_uri", "HTTP_URI:request_firstline_uri"); - LOGFIELDS.put("request_firstline_uri_ref", "HTTP_REF:request_firstline_uri_ref"); - LOGFIELDS.put("request_receive_time_year", "TIME_YEAR:request_receive_time_year"); - LOGFIELDS.put("request_firstline", "HTTP_FIRSTLINE:request_firstline"); - LOGFIELDS.put("request_user-agent", "HTTP_USERAGENT:request_user-agent"); - LOGFIELDS.put("request_cookies", "HTTP_COOKIE:request_cookies"); - LOGFIELDS.put("server_process_time", "MICROSECONDS:server_process_time"); - LOGFIELDS.put("request_cookies_$", "HTTP_COOKIE:request_cookies_$"); - LOGFIELDS.put("server_environment_$", "VARIABLE:server_environment_$"); - LOGFIELDS.put("server_filename", "FILENAME:server_filename"); - LOGFIELDS.put("request_protocol", "PROTOCOL:request_protocol"); - LOGFIELDS.put("request_header_", "HTTP_HEADER:request_header_"); - LOGFIELDS.put("connection_keepalivecount", "NUMBER:connection_keepalivecount"); - LOGFIELDS.put("connection_client_logname", "NUMBER:connection_client_logname"); - LOGFIELDS.put("request_errorlogid", "STRING:request_errorlogid"); - LOGFIELDS.put("request_method", "HTTP_METHOD:request_method"); - LOGFIELDS.put("server_module_note_$", "STRING:server_module_note_$"); - LOGFIELDS.put("response_header_$", "HTTP_HEADER:response_header_$"); - LOGFIELDS.put("request_server_port_canonical", "PORT:request_server_port_canonical"); - LOGFIELDS.put("connection_server_port_canonical", "PORT:connection_server_port_canonical"); - LOGFIELDS.put("connection_server_port", "PORT:connection_server_port"); - LOGFIELDS.put("connection_client_port", "PORT:connection_client_port"); - LOGFIELDS.put("connection_server_child_processid", "NUMBER:connection_server_child_processid"); - LOGFIELDS.put("connection_server_child_threadid", "NUMBER:connection_server_child_threadid"); - LOGFIELDS.put("connection_server_child_hexthreadid", "NUMBER:connection_server_child_hexthreadid"); - LOGFIELDS.put("request_querystring", "HTTP_QUERYSTRING:request_querystring"); - LOGFIELDS.put("request_handler", "STRING:request_handler"); - LOGFIELDS.put("request_status_original", "STRING:request_status_original"); - LOGFIELDS.put("request_status_last", "STRING:request_status_last"); - LOGFIELDS.put("request_receive_time_begin_msec", "TIME_EPOCH:request_receive_time_begin_msec"); - LOGFIELDS.put("request_receive_time_end_msec", "TIME_EPOCH:request_receive_time_end_msec"); - LOGFIELDS.put("request_receive_time_begin_usec", "TIME_EPOCH_USEC:request_receive_time_begin_usec"); - LOGFIELDS.put("request_receive_time_begin_usec", "TIME_EPOCH_USEC:request_receive_time_begin_usec"); - LOGFIELDS.put("request_receive_time_end_usec", "TIME_EPOCH_USEC:request_receive_time_end_usec"); - LOGFIELDS.put("request_receive_time_begin_msec_frac", "TIME_EPOCH:request_receive_time_begin_msec_frac"); - LOGFIELDS.put("request_receive_time_begin_msec_frac", "TIME_EPOCH:request_receive_time_begin_msec_frac"); - LOGFIELDS.put("request_receive_time_end_msec_frac", "TIME_EPOCH:request_receive_time_end_msec_frac"); - LOGFIELDS.put("request_receive_time_begin_usec_frac", "TIME_EPOCH_USEC_FRAC:request_receive_time_begin_usec_frac"); - LOGFIELDS.put("request_receive_time_begin_usec_frac", "TIME_EPOCH_USEC_FRAC:request.receive.time.begin.usec_frac"); - LOGFIELDS.put("request_receive_time_end_usec_frac", "TIME_EPOCH_USEC_FRAC:request_receive_time_end_usec_frac"); - LOGFIELDS.put("response_server_processing_time", "SECONDS:response_server_processing_time"); - LOGFIELDS.put("connection_client_user", "STRING:connection_client_user"); - LOGFIELDS.put("request_urlpath", "URI:request_urlpath"); - LOGFIELDS.put("connection_server_name_canonical", "STRING:connection_server_name_canonical"); - LOGFIELDS.put("connection_server_name", "STRING:connection_server_name"); - LOGFIELDS.put("response_connection_status", "HTTP_CONNECTSTATUS:response_connection_status"); - LOGFIELDS.put("request_bytes", "BYTES:request_bytes"); - LOGFIELDS.put("response_bytes", "BYTES:response_bytes"); - } + public static final HashMap<String, String> LOGFIELDS = new HashMap<String, String>(); - //Map map = Collections.synchronizedMap(LOGFIELDS); + static { + LOGFIELDS.put("connection.client.ip", "IP:connection.client.ip"); + LOGFIELDS.put("connection.client.ip.last", "IP:connection.client.ip.last"); + LOGFIELDS.put("connection.client.ip.original", "IP:connection.client.ip.original"); + LOGFIELDS.put("connection.client.ip.last", "IP:connection.client.ip.last"); + LOGFIELDS.put("connection.client.peerip", "IP:connection.client.peerip"); + LOGFIELDS.put("connection.client.peerip.last", "IP:connection.client.peerip.last"); + LOGFIELDS.put("connection.client.peerip.original", "IP:connection.client.peerip.original"); + LOGFIELDS.put("connection.client.peerip.last", "IP:connection.client.peerip.last"); + LOGFIELDS.put("connection.server.ip", "IP:connection.server.ip"); + LOGFIELDS.put("connection.server.ip.last", "IP:connection.server.ip.last"); + LOGFIELDS.put("connection.server.ip.original", "IP:connection.server.ip.original"); + LOGFIELDS.put("connection.server.ip.last", "IP:connection.server.ip.last"); + LOGFIELDS.put("response.body.bytes", "BYTES:response.body.bytes"); + LOGFIELDS.put("response.body.bytes.last", "BYTES:response.body.bytes.last"); + LOGFIELDS.put("response.body.bytes.original", "BYTES:response.body.bytes.original"); + LOGFIELDS.put("response.body.bytes.last", "BYTES:response.body.bytes.last"); + LOGFIELDS.put("response.body.bytesclf", "BYTES:response.body.bytesclf"); + LOGFIELDS.put("response.body.bytes", "BYTESCLF:response.body.bytes"); + LOGFIELDS.put("response.body.bytes.last", "BYTESCLF:response.body.bytes.last"); + LOGFIELDS.put("response.body.bytes.original", "BYTESCLF:response.body.bytes.original"); + LOGFIELDS.put("response.body.bytes.last", "BYTESCLF:response.body.bytes.last"); + LOGFIELDS.put("request.cookies.foobar", "HTTP.COOKIE:request.cookies.foobar"); + LOGFIELDS.put("server.environment.foobar", "VARIABLE:server.environment.foobar"); + LOGFIELDS.put("server.filename", "FILENAME:server.filename"); + LOGFIELDS.put("server.filename.last", "FILENAME:server.filename.last"); + LOGFIELDS.put("server.filename.original", "FILENAME:server.filename.original"); + LOGFIELDS.put("server.filename.last", "FILENAME:server.filename.last"); + LOGFIELDS.put("connection.client.host", "IP:connection.client.host"); + LOGFIELDS.put("connection.client.host.last", "IP:connection.client.host.last"); + LOGFIELDS.put("connection.client.host.original", "IP:connection.client.host.original"); + LOGFIELDS.put("connection.client.host.last", "IP:connection.client.host.last"); + LOGFIELDS.put("request.protocol", "PROTOCOL:request.protocol"); + LOGFIELDS.put("request.protocol.last", "PROTOCOL:request.protocol.last"); + LOGFIELDS.put("request.protocol.original", "PROTOCOL:request.protocol.original"); + LOGFIELDS.put("request.protocol.last", "PROTOCOL:request.protocol.last"); + LOGFIELDS.put("request.header.foobar", "HTTP.HEADER:request.header.foobar"); + LOGFIELDS.put("request.trailer.foobar", "HTTP.TRAILER:request.trailer.foobar"); + LOGFIELDS.put("connection.keepalivecount", "NUMBER:connection.keepalivecount"); + LOGFIELDS.put("connection.keepalivecount.last", "NUMBER:connection.keepalivecount.last"); + LOGFIELDS.put("connection.keepalivecount.original", "NUMBER:connection.keepalivecount.original"); + LOGFIELDS.put("connection.keepalivecount.last", "NUMBER:connection.keepalivecount.last"); + LOGFIELDS.put("connection.client.logname", "NUMBER:connection.client.logname"); + LOGFIELDS.put("connection.client.logname.last", "NUMBER:connection.client.logname.last"); + LOGFIELDS.put("connection.client.logname.original", "NUMBER:connection.client.logname.original"); + LOGFIELDS.put("connection.client.logname.last", "NUMBER:connection.client.logname.last"); + LOGFIELDS.put("request.errorlogid", "STRING:request.errorlogid"); + LOGFIELDS.put("request.errorlogid.last", "STRING:request.errorlogid.last"); + LOGFIELDS.put("request.errorlogid.original", "STRING:request.errorlogid.original"); + LOGFIELDS.put("request.errorlogid.last", "STRING:request.errorlogid.last"); + LOGFIELDS.put("request.method", "HTTP.METHOD:request.method"); + LOGFIELDS.put("request.method.last", "HTTP.METHOD:request.method.last"); + LOGFIELDS.put("request.method.original", "HTTP.METHOD:request.method.original"); + LOGFIELDS.put("request.method.last", "HTTP.METHOD:request.method.last"); + LOGFIELDS.put("server.module_note.foobar", "STRING:server.module_note.foobar"); + LOGFIELDS.put("response.header.foobar", "HTTP.HEADER:response.header.foobar"); + LOGFIELDS.put("response.trailer.foobar", "HTTP.TRAILER:response.trailer.foobar"); + LOGFIELDS.put("request.server.port.canonical", "PORT:request.server.port.canonical"); + LOGFIELDS.put("request.server.port.canonical.last", "PORT:request.server.port.canonical.last"); + LOGFIELDS.put("request.server.port.canonical.original", "PORT:request.server.port.canonical.original"); + LOGFIELDS.put("request.server.port.canonical.last", "PORT:request.server.port.canonical.last"); + LOGFIELDS.put("connection.server.port.canonical", "PORT:connection.server.port.canonical"); + LOGFIELDS.put("connection.server.port.canonical.last", "PORT:connection.server.port.canonical.last"); + LOGFIELDS.put("connection.server.port.canonical.original", "PORT:connection.server.port.canonical.original"); + LOGFIELDS.put("connection.server.port.canonical.last", "PORT:connection.server.port.canonical.last"); + LOGFIELDS.put("connection.server.port", "PORT:connection.server.port"); + LOGFIELDS.put("connection.server.port.last", "PORT:connection.server.port.last"); + LOGFIELDS.put("connection.server.port.original", "PORT:connection.server.port.original"); + LOGFIELDS.put("connection.server.port.last", "PORT:connection.server.port.last"); + LOGFIELDS.put("connection.client.port", "PORT:connection.client.port"); + LOGFIELDS.put("connection.client.port.last", "PORT:connection.client.port.last"); + LOGFIELDS.put("connection.client.port.original", "PORT:connection.client.port.original"); + LOGFIELDS.put("connection.client.port.last", "PORT:connection.client.port.last"); + LOGFIELDS.put("connection.server.child.processid", "NUMBER:connection.server.child.processid"); + LOGFIELDS.put("connection.server.child.processid.last", "NUMBER:connection.server.child.processid.last"); + LOGFIELDS.put("connection.server.child.processid.original", "NUMBER:connection.server.child.processid.original"); + LOGFIELDS.put("connection.server.child.processid.last", "NUMBER:connection.server.child.processid.last"); + LOGFIELDS.put("connection.server.child.processid", "NUMBER:connection.server.child.processid"); + LOGFIELDS.put("connection.server.child.processid.last", "NUMBER:connection.server.child.processid.last"); + LOGFIELDS.put("connection.server.child.processid.original", "NUMBER:connection.server.child.processid.original"); + LOGFIELDS.put("connection.server.child.processid.last", "NUMBER:connection.server.child.processid.last"); + LOGFIELDS.put("connection.server.child.threadid", "NUMBER:connection.server.child.threadid"); + LOGFIELDS.put("connection.server.child.threadid.last", "NUMBER:connection.server.child.threadid.last"); + LOGFIELDS.put("connection.server.child.threadid.original", "NUMBER:connection.server.child.threadid.original"); + LOGFIELDS.put("connection.server.child.threadid.last", "NUMBER:connection.server.child.threadid.last"); + LOGFIELDS.put("connection.server.child.hexthreadid", "NUMBER:connection.server.child.hexthreadid"); + LOGFIELDS.put("connection.server.child.hexthreadid.last", "NUMBER:connection.server.child.hexthreadid.last"); + LOGFIELDS.put("connection.server.child.hexthreadid.original", "NUMBER:connection.server.child.hexthreadid.original"); + LOGFIELDS.put("connection.server.child.hexthreadid.last", "NUMBER:connection.server.child.hexthreadid.last"); + LOGFIELDS.put("request.querystring", "HTTP.QUERYSTRING:request.querystring"); + LOGFIELDS.put("request.querystring.last", "HTTP.QUERYSTRING:request.querystring.last"); + LOGFIELDS.put("request.querystring.original", "HTTP.QUERYSTRING:request.querystring.original"); + LOGFIELDS.put("request.querystring.last", "HTTP.QUERYSTRING:request.querystring.last"); + LOGFIELDS.put("request.firstline", "HTTP.FIRSTLINE:request.firstline"); + LOGFIELDS.put("request.firstline.original", "HTTP.FIRSTLINE:request.firstline.original"); + LOGFIELDS.put("request.firstline.original", "HTTP.FIRSTLINE:request.firstline.original"); + LOGFIELDS.put("request.firstline.last", "HTTP.FIRSTLINE:request.firstline.last"); + LOGFIELDS.put("request.handler", "STRING:request.handler"); + LOGFIELDS.put("request.handler.last", "STRING:request.handler.last"); + LOGFIELDS.put("request.handler.original", "STRING:request.handler.original"); + LOGFIELDS.put("request.handler.last", "STRING:request.handler.last"); + LOGFIELDS.put("request.status", "STRING:request.status"); + LOGFIELDS.put("request.status.original", "STRING:request.status.original"); + LOGFIELDS.put("request.status.original", "STRING:request.status.original"); + LOGFIELDS.put("request.status.last", "STRING:request.status.last"); + LOGFIELDS.put("request.receive.time", "TIME.STAMP:request.receive.time"); + LOGFIELDS.put("request.receive.time.last", "TIME.STAMP:request.receive.time.last"); + LOGFIELDS.put("request.receive.time.original", "TIME.STAMP:request.receive.time.original"); + LOGFIELDS.put("request.receive.time.last", "TIME.STAMP:request.receive.time.last"); + LOGFIELDS.put("request.receive.time.year", "TIME.YEAR:request.receive.time.year"); + LOGFIELDS.put("request.receive.time.begin.year", "TIME.YEAR:request.receive.time.begin.year"); + LOGFIELDS.put("request.receive.time.end.year", "TIME.YEAR:request.receive.time.end.year"); + LOGFIELDS.put("request.receive.time.sec", "TIME.SECONDS:request.receive.time.sec"); + LOGFIELDS.put("request.receive.time.sec", "TIME.SECONDS:request.receive.time.sec"); + LOGFIELDS.put("request.receive.time.sec.original", "TIME.SECONDS:request.receive.time.sec.original"); + LOGFIELDS.put("request.receive.time.sec.last", "TIME.SECONDS:request.receive.time.sec.last"); + LOGFIELDS.put("request.receive.time.begin.sec", "TIME.SECONDS:request.receive.time.begin.sec"); + LOGFIELDS.put("request.receive.time.begin.sec.last", "TIME.SECONDS:request.receive.time.begin.sec.last"); + LOGFIELDS.put("request.receive.time.begin.sec.original", "TIME.SECONDS:request.receive.time.begin.sec.original"); + LOGFIELDS.put("request.receive.time.begin.sec.last", "TIME.SECONDS:request.receive.time.begin.sec.last"); + LOGFIELDS.put("request.receive.time.end.sec", "TIME.SECONDS:request.receive.time.end.sec"); + LOGFIELDS.put("request.receive.time.end.sec.last", "TIME.SECONDS:request.receive.time.end.sec.last"); + LOGFIELDS.put("request.receive.time.end.sec.original", "TIME.SECONDS:request.receive.time.end.sec.original"); + LOGFIELDS.put("request.receive.time.end.sec.last", "TIME.SECONDS:request.receive.time.end.sec.last"); + LOGFIELDS.put("request.receive.time.begin.msec", "TIME.EPOCH:request.receive.time.begin.msec"); + LOGFIELDS.put("request.receive.time.msec", "TIME.EPOCH:request.receive.time.msec"); + LOGFIELDS.put("request.receive.time.msec.last", "TIME.EPOCH:request.receive.time.msec.last"); + LOGFIELDS.put("request.receive.time.msec.original", "TIME.EPOCH:request.receive.time.msec.original"); + LOGFIELDS.put("request.receive.time.msec.last", "TIME.EPOCH:request.receive.time.msec.last"); + LOGFIELDS.put("request.receive.time.begin.msec", "TIME.EPOCH:request.receive.time.begin.msec"); + LOGFIELDS.put("request.receive.time.begin.msec.last", "TIME.EPOCH:request.receive.time.begin.msec.last"); + LOGFIELDS.put("request.receive.time.begin.msec.original", "TIME.EPOCH:request.receive.time.begin.msec.original"); + LOGFIELDS.put("request.receive.time.begin.msec.last", "TIME.EPOCH:request.receive.time.begin.msec.last"); + LOGFIELDS.put("request.receive.time.end.msec", "TIME.EPOCH:request.receive.time.end.msec"); + LOGFIELDS.put("request.receive.time.end.msec.last", "TIME.EPOCH:request.receive.time.end.msec.last"); + LOGFIELDS.put("request.receive.time.end.msec.original", "TIME.EPOCH:request.receive.time.end.msec.original"); + LOGFIELDS.put("request.receive.time.end.msec.last", "TIME.EPOCH:request.receive.time.end.msec.last"); + LOGFIELDS.put("request.receive.time.begin.usec", "TIME.EPOCH.USEC:request.receive.time.begin.usec"); + LOGFIELDS.put("request.receive.time.usec", "TIME.EPOCH.USEC:request.receive.time.usec"); + LOGFIELDS.put("request.receive.time.usec.last", "TIME.EPOCH.USEC:request.receive.time.usec.last"); + LOGFIELDS.put("request.receive.time.usec.original", "TIME.EPOCH.USEC:request.receive.time.usec.original"); + LOGFIELDS.put("request.receive.time.usec.last", "TIME.EPOCH.USEC:request.receive.time.usec.last"); + LOGFIELDS.put("request.receive.time.begin.usec", "TIME.EPOCH.USEC:request.receive.time.begin.usec"); + LOGFIELDS.put("request.receive.time.begin.usec.last", "TIME.EPOCH.USEC:request.receive.time.begin.usec.last"); + LOGFIELDS.put("request.receive.time.begin.usec.original", "TIME.EPOCH.USEC:request.receive.time.begin.usec.original"); + LOGFIELDS.put("request.receive.time.begin.usec.last", "TIME.EPOCH.USEC:request.receive.time.begin.usec.last"); + LOGFIELDS.put("request.receive.time.end.usec", "TIME.EPOCH.USEC:request.receive.time.end.usec"); + LOGFIELDS.put("request.receive.time.end.usec.last", "TIME.EPOCH.USEC:request.receive.time.end.usec.last"); + LOGFIELDS.put("request.receive.time.end.usec.original", "TIME.EPOCH.USEC:request.receive.time.end.usec.original"); + LOGFIELDS.put("request.receive.time.end.usec.last", "TIME.EPOCH.USEC:request.receive.time.end.usec.last"); + LOGFIELDS.put("request.receive.time.begin.msec_frac", "TIME.EPOCH:request.receive.time.begin.msec_frac"); + LOGFIELDS.put("request.receive.time.msec_frac", "TIME.EPOCH:request.receive.time.msec_frac"); + LOGFIELDS.put("request.receive.time.msec_frac.last", "TIME.EPOCH:request.receive.time.msec_frac.last"); + LOGFIELDS.put("request.receive.time.msec_frac.original", "TIME.EPOCH:request.receive.time.msec_frac.original"); + LOGFIELDS.put("request.receive.time.msec_frac.last", "TIME.EPOCH:request.receive.time.msec_frac.last"); + LOGFIELDS.put("request.receive.time.begin.msec_frac", "TIME.EPOCH:request.receive.time.begin.msec_frac"); + LOGFIELDS.put("request.receive.time.begin.msec_frac.last", "TIME.EPOCH:request.receive.time.begin.msec_frac.last"); + LOGFIELDS.put("request.receive.time.begin.msec_frac.original", "TIME.EPOCH:request.receive.time.begin.msec_frac.original"); + LOGFIELDS.put("request.receive.time.begin.msec_frac.last", "TIME.EPOCH:request.receive.time.begin.msec_frac.last"); + LOGFIELDS.put("request.receive.time.end.msec_frac", "TIME.EPOCH:request.receive.time.end.msec_frac"); + LOGFIELDS.put("request.receive.time.end.msec_frac.last", "TIME.EPOCH:request.receive.time.end.msec_frac.last"); + LOGFIELDS.put("request.receive.time.end.msec_frac.original", "TIME.EPOCH:request.receive.time.end.msec_frac.original"); + LOGFIELDS.put("request.receive.time.end.msec_frac.last", "TIME.EPOCH:request.receive.time.end.msec_frac.last"); + LOGFIELDS.put("request.receive.time.begin.usec_frac", "FRAC:request.receive.time.begin.usec_frac"); + LOGFIELDS.put("request.receive.time.usec_frac", "FRAC:request.receive.time.usec_frac"); + LOGFIELDS.put("request.receive.time.usec_frac.last", "FRAC:request.receive.time.usec_frac.last"); + LOGFIELDS.put("request.receive.time.usec_frac.original", "FRAC:request.receive.time.usec_frac.original"); + LOGFIELDS.put("request.receive.time.usec_frac.last", "FRAC:request.receive.time.usec_frac.last"); + LOGFIELDS.put("request.receive.time.begin.usec_frac", "FRAC:request.receive.time.begin.usec_frac"); + LOGFIELDS.put("request.receive.time.begin.usec_frac.last", "FRAC:request.receive.time.begin.usec_frac.last"); + LOGFIELDS.put("request.receive.time.begin.usec_frac.original", "FRAC:request.receive.time.begin.usec_frac.original"); + LOGFIELDS.put("request.receive.time.begin.usec_frac.last", "FRAC:request.receive.time.begin.usec_frac.last"); + LOGFIELDS.put("request.receive.time.end.usec_frac", "FRAC:request.receive.time.end.usec_frac"); + LOGFIELDS.put("request.receive.time.end.usec_frac.last", "FRAC:request.receive.time.end.usec_frac.last"); + LOGFIELDS.put("request.receive.time.end.usec_frac.original", "FRAC:request.receive.time.end.usec_frac.original"); + LOGFIELDS.put("request.receive.time.end.usec_frac.last", "FRAC:request.receive.time.end.usec_frac.last"); + LOGFIELDS.put("response.server.processing.time", "SECONDS:response.server.processing.time"); + LOGFIELDS.put("response.server.processing.time.original", "SECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.original", "SECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.last", "SECONDS:response.server.processing.time.last"); + LOGFIELDS.put("server.process.time", "MICROSECONDS:server.process.time"); + LOGFIELDS.put("response.server.processing.time", "MICROSECONDS:response.server.processing.time"); + LOGFIELDS.put("response.server.processing.time.original", "MICROSECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.original", "MICROSECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.last", "MICROSECONDS:response.server.processing.time.last"); + LOGFIELDS.put("response.server.processing.time", "MICROSECONDS:response.server.processing.time"); + LOGFIELDS.put("response.server.processing.time.original", "MICROSECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.original", "MICROSECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.last", "MICROSECONDS:response.server.processing.time.last"); + LOGFIELDS.put("response.server.processing.time", "MILLISECONDS:response.server.processing.time"); + LOGFIELDS.put("response.server.processing.time.original", "MILLISECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.original", "MILLISECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.last", "MILLISECONDS:response.server.processing.time.last"); + LOGFIELDS.put("response.server.processing.time", "SECONDS:response.server.processing.time"); + LOGFIELDS.put("response.server.processing.time.original", "SECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.original", "SECONDS:response.server.processing.time.original"); + LOGFIELDS.put("response.server.processing.time.last", "SECONDS:response.server.processing.time.last"); + LOGFIELDS.put("connection.client.user", "STRING:connection.client.user"); + LOGFIELDS.put("connection.client.user.last", "STRING:connection.client.user.last"); + LOGFIELDS.put("connection.client.user.original", "STRING:connection.client.user.original"); + LOGFIELDS.put("connection.client.user.last", "STRING:connection.client.user.last"); + LOGFIELDS.put("request.urlpath", "URI:request.urlpath"); + LOGFIELDS.put("request.urlpath.original", "URI:request.urlpath.original"); + LOGFIELDS.put("request.urlpath.original", "URI:request.urlpath.original"); + LOGFIELDS.put("request.urlpath.last", "URI:request.urlpath.last"); + LOGFIELDS.put("connection.server.name.canonical", "STRING:connection.server.name.canonical"); + LOGFIELDS.put("connection.server.name.canonical.last", "STRING:connection.server.name.canonical.last"); + LOGFIELDS.put("connection.server.name.canonical.original", "STRING:connection.server.name.canonical.original"); + LOGFIELDS.put("connection.server.name.canonical.last", "STRING:connection.server.name.canonical.last"); + LOGFIELDS.put("connection.server.name", "STRING:connection.server.name"); + LOGFIELDS.put("connection.server.name.last", "STRING:connection.server.name.last"); + LOGFIELDS.put("connection.server.name.original", "STRING:connection.server.name.original"); + LOGFIELDS.put("connection.server.name.last", "STRING:connection.server.name.last"); + LOGFIELDS.put("response.connection.status", "HTTP.CONNECTSTATUS:response.connection.status"); + LOGFIELDS.put("response.connection.status.last", "HTTP.CONNECTSTATUS:response.connection.status.last"); + LOGFIELDS.put("response.connection.status.original", "HTTP.CONNECTSTATUS:response.connection.status.original"); + LOGFIELDS.put("response.connection.status.last", "HTTP.CONNECTSTATUS:response.connection.status.last"); + LOGFIELDS.put("request.bytes", "BYTES:request.bytes"); + LOGFIELDS.put("request.bytes.last", "BYTES:request.bytes.last"); + LOGFIELDS.put("request.bytes.original", "BYTES:request.bytes.original"); + LOGFIELDS.put("request.bytes.last", "BYTES:request.bytes.last"); + LOGFIELDS.put("response.bytes", "BYTES:response.bytes"); + LOGFIELDS.put("response.bytes.last", "BYTES:response.bytes.last"); + LOGFIELDS.put("response.bytes.original", "BYTES:response.bytes.original"); + LOGFIELDS.put("response.bytes.last", "BYTES:response.bytes.last"); + LOGFIELDS.put("total.bytes", "BYTES:total.bytes"); + LOGFIELDS.put("total.bytes.last", "BYTES:total.bytes.last"); + LOGFIELDS.put("total.bytes.original", "BYTES:total.bytes.original"); + LOGFIELDS.put("total.bytes.last", "BYTES:total.bytes.last"); + LOGFIELDS.put("request.cookies", "HTTP.COOKIES:request.cookies"); + LOGFIELDS.put("request.cookies.last", "HTTP.COOKIES:request.cookies.last"); + LOGFIELDS.put("request.cookies.original", "HTTP.COOKIES:request.cookies.original"); + LOGFIELDS.put("request.cookies.last", "HTTP.COOKIES:request.cookies.last"); + LOGFIELDS.put("response.cookies", "HTTP.SETCOOKIES:response.cookies"); + LOGFIELDS.put("response.cookies.last", "HTTP.SETCOOKIES:response.cookies.last"); + LOGFIELDS.put("response.cookies.original", "HTTP.SETCOOKIES:response.cookies.original"); + LOGFIELDS.put("response.cookies.last", "HTTP.SETCOOKIES:response.cookies.last"); + LOGFIELDS.put("request.user-agent", "HTTP.USERAGENT:request.user-agent"); + LOGFIELDS.put("request.user-agent.last", "HTTP.USERAGENT:request.user-agent.last"); + LOGFIELDS.put("request.user-agent.original", "HTTP.USERAGENT:request.user-agent.original"); + LOGFIELDS.put("request.user-agent.last", "HTTP.USERAGENT:request.user-agent.last"); + LOGFIELDS.put("request.referer", "HTTP.URI:request.referer"); + LOGFIELDS.put("request.referer.last", "HTTP.URI:request.referer.last"); + LOGFIELDS.put("request.referer.original", "HTTP.URI:request.referer.original"); + LOGFIELDS.put("request.referer.last", "HTTP.URI:request.referer.last"); + } public HttpdParser(final MapWriter mapWriter, final DrillBuf managedBuffer, final String logFormat, - final String timestampFormat, final Map<String, String> fieldMapping) - throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException { + final String timestampFormat, final Map<String, String> fieldMapping) + throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException { Preconditions.checkArgument(logFormat != null && !logFormat.trim().isEmpty(), "logFormat cannot be null or empty"); - this.record = new HttpdLogRecord(managedBuffer); + this.record = new HttpdLogRecord(managedBuffer, timestampFormat); this.parser = new HttpdLoglineParser<>(HttpdLogRecord.class, logFormat, timestampFormat); setupParser(mapWriter, logFormat, fieldMapping); @@ -167,7 +317,6 @@ public class HttpdParser { * We do not expose the underlying parser or the record which is used to manage the writers. * * @param line log line to tear apart. - * * @throws DissectionFailure * @throws InvalidDissectorException * @throws MissingDissectorsException @@ -181,7 +330,7 @@ public class HttpdParser { * In order to define a type remapping the format of the field configuration will look like: <br/> * HTTP.URI:request.firstline.uri.query.[parameter name] <br/> * - * @param parser Add type remapping to this parser instance. + * @param parser Add type remapping to this parser instance. * @param fieldName request.firstline.uri.query.[parameter_name] * @param fieldType HTTP.URI, etc.. */ @@ -198,11 +347,17 @@ public class HttpdParser { * @param drillFieldName name to be cleansed. * @return */ - public static String parserFormattedFieldName(final String drillFieldName) { - String tempFieldName; - tempFieldName = LOGFIELDS.get(drillFieldName); - return tempFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_"); + public static String parserFormattedFieldName(String drillFieldName) { + //The Useragent fields contain a dash which causes potential problems if the field name is not escaped properly + //This removes the dash + if (drillFieldName.contains("useragent")) { + drillFieldName = drillFieldName.replace("useragent", "user-agent"); + } + + String tempFieldName; + tempFieldName = LOGFIELDS.get(drillFieldName); + return tempFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_"); } /** @@ -213,19 +368,24 @@ public class HttpdParser { * @param parserFieldName name to be cleansed. * @return */ - public static String drillFormattedFieldName(final String parserFieldName) { + public static String drillFormattedFieldName(String parserFieldName) { - if (parserFieldName.contains(":") ) { - String[] fieldPart= parserFieldName.split(":"); - return fieldPart[1].replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR); - } - else{ + //The Useragent fields contain a dash which causes potential problems if the field name is not escaped properly + //This removes the dash + if (parserFieldName.contains("user-agent")) { + parserFieldName = parserFieldName.replace("user-agent", "useragent"); + } + + if (parserFieldName.contains(":")) { + String[] fieldPart = parserFieldName.split(":"); + return fieldPart[1].replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR); + } else { return parserFieldName.replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR); } } private void setupParser(final MapWriter mapWriter, final String logFormat, final Map<String, String> fieldMapping) - throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException { + throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException { /** * If the user has selected fields, then we will use them to configure the parser because this would be the most @@ -236,8 +396,7 @@ public class HttpdParser { if (fieldMapping != null && !fieldMapping.isEmpty()) { LOG.debug("Using fields defined by user."); requestedPaths = fieldMapping; - } - else { + } else { /** * Use all possible paths that the parser has determined from the specified log format. */ @@ -255,7 +414,6 @@ public class HttpdParser { */ Parser<Object> dummy = new HttpdLoglineParser<>(Object.class, logFormat); dummy.addParseTarget(String.class.getMethod("indexOf", String.class), allParserPaths); - for (final Map.Entry<String, String> entry : requestedPaths.entrySet()) { final EnumSet<Casts> casts; @@ -270,10 +428,8 @@ public class HttpdParser { final String[] pieces = entry.getValue().split(":"); addTypeRemapping(parser, pieces[1], pieces[0]); - casts = Casts.STRING_ONLY; - } - else { + } else { casts = dummy.getCasts(entry.getValue()); } @@ -281,4 +437,4 @@ public class HttpdParser { record.addField(parser, mapWriter, casts, entry.getValue(), entry.getKey()); } } -}
\ No newline at end of file +} |