1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
{
"storage":{
dfs: {
type: "file",
connection: "file:///",
workspaces: {
"root" : {
location: "/",
writable: false,
allowAccessOutsideWorkspace: false
},
"tmp" : {
location: "/tmp",
writable: true,
allowAccessOutsideWorkspace: false
}
},
formats: {
"psv" : {
type: "text",
extensions: [ "tbl" ],
delimiter: "|"
},
"csv" : {
type: "text",
extensions: [ "csv" ],
delimiter: ","
},
"tsv" : {
type: "text",
extensions: [ "tsv" ],
delimiter: "\t"
},
"httpd" : {
type: "httpd",
logFormat: "%h %t \"%r\" %>s %b \"%{Referer}i\""
/* timestampFormat: "dd/MMM/yyyy:HH:mm:ss ZZ" */
},
"parquet" : {
type: "parquet"
},
"json" : {
type: "json",
extensions: [ "json" ]
},
"pcap" : {
type: "pcap"
},
"pcapng" : {
type: "pcapng"
},
"avro" : {
type: "avro"
},
"sequencefile": {
type : "sequencefile",
extensions: [ "seq" ]
},
"csvh" : {
type: "text",
extensions: [ "csvh" ],
delimiter: ",",
extractHeader: true
},
"image" : {
type: "image",
extensions: [
"jpg", "jpeg", "jpe", "tif", "tiff", "dng", "psd", "png", "bmp", "gif",
"ico", "pcx", "wav", "wave", "avi", "webp", "mov", "mp4", "m4a", "m4p",
"m4b", "m4r", "m4v", "3gp", "3g2", "eps", "epsf", "epsi", "ai", "arw",
"crw", "cr2", "nef", "orf", "raf", "rw2", "rwl", "srw", "x3f"
]
}
},
enabled : true
},
s3: {
type: "file",
connection: "s3a://my.bucket.location.com",
config : {
"fs.s3a.access.key": "ID",
"fs.s3a.secret.key": "SECRET"
},
workspaces: {
"root" : {
location: "/",
writable: false
},
"tmp" : {
location: "/tmp",
writable: true
}
},
formats: {
"psv" : {
type: "text",
extensions: [ "tbl" ],
delimiter: "|"
},
"csv" : {
type: "text",
extensions: [ "csv" ],
delimiter: ","
},
"tsv" : {
type: "text",
extensions: [ "tsv" ],
delimiter: "\t"
},
"parquet" : {
type: "parquet"
},
"json" : {
type: "json",
extensions: [ "json" ]
},
"avro" : {
type: "avro"
},
"sequencefile": {
type : "sequencefile",
extensions: [ "seq" ]
},
"csvh" : {
type: "text",
extensions: [ "csvh" ],
delimiter: ",",
extractHeader: true
}
},
enabled : false
},
cp: {
type: "file",
connection: "classpath:///",
formats: {
"csv" : {
type: "text",
extensions: [ "csv" ],
delimiter: ","
},
"tsv" : {
type: "text",
extensions: [ "tsv" ],
delimiter: "\t"
},
"json" : {
type: "json",
extensions: [ "json" ]
},
"parquet" : {
type: "parquet"
},
"avro" : {
type: "avro"
},
"csvh" : {
type: "text",
extensions: [ "csvh" ],
delimiter: ",",
extractHeader: true
},
"image" : {
type: "image",
extensions: [
"jpg", "jpeg", "jpe", "tif", "tiff", "dng", "psd", "png", "bmp", "gif",
"ico", "pcx", "wav", "wave", "avi", "webp", "mov", "mp4", "m4a", "m4p",
"m4b", "m4r", "m4v", "3gp", "3g2", "eps", "epsf", "epsi", "ai", "arw",
"crw", "cr2", "nef", "orf", "raf", "rw2", "rwl", "srw", "x3f"
]
}
},
enabled : true
}
}
}
|
