1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
|
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.store;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PositionedReadable;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Progressable;
/**
* This class provides a Syncable local extension of the hadoop FileSystem
*/
public class LocalSyncableFileSystem extends FileSystem {
@Override
public URI getUri() {
try {
return new URI("drill-local:///");
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
@Override
public FSDataInputStream open(Path path, int i) throws IOException {
return new FSDataInputStream(new LocalInputStream(path));
}
@Override
public FSDataOutputStream create(Path path, FsPermission fsPermission, boolean b, int i, short i2, long l, Progressable progressable) throws IOException {
return new FSDataOutputStream(new LocalSyncableOutputStream(path));
}
@Override
public FSDataOutputStream append(Path path, int i, Progressable progressable) throws IOException {
throw new IOException("Append is not supported in LocalSyncableFilesystem");
}
@Override
public boolean rename(Path path, Path path2) throws IOException {
throw new IOException("Rename not supported");
}
@Override
public boolean delete(Path path) throws IOException {
File file = new File(path.toString());
return file.delete();
}
@Override
public boolean delete(Path path, boolean b) throws IOException {
File file = new File(path.toString());
if (b) {
if (file.isDirectory()) {
FileUtils.deleteDirectory(file);
} else {
file.delete();
}
} else if (file.isDirectory()) {
throw new IOException("Cannot delete directory");
}
file.delete();
return true;
}
@Override
public FileStatus[] listStatus(Path path) throws IOException {
throw new IOException("listStatus not supported");
}
@Override
public void setWorkingDirectory(Path path) {
}
@Override
public Path getWorkingDirectory() {
return null;
}
@Override
public boolean mkdirs(Path path, FsPermission fsPermission) throws IOException {
return new File(path.toString()).mkdirs();
}
@Override
public FileStatus getFileStatus(Path path) throws IOException {
File file = new File(Path.getPathWithoutSchemeAndAuthority(path).toString());
return new FileStatus(file.length(), file.isDirectory(), 1, 0, file.lastModified(), path);
}
public class LocalSyncableOutputStream extends OutputStream implements Syncable {
private FileOutputStream fos;
private BufferedOutputStream output;
public LocalSyncableOutputStream(Path path) throws FileNotFoundException {
File dir = new File(path.getParent().toString());
if (!dir.exists()) {
boolean success = dir.mkdirs();
if (!success) {
throw new FileNotFoundException("failed to create parent directory");
}
}
fos = new FileOutputStream(new File(path.toString()));
output = new BufferedOutputStream(fos, 64*1024);
}
@Override
public void sync() throws IOException {
output.flush();
fos.getFD().sync();
}
@Override
public void hsync() throws IOException {
output.flush();
fos.getFD().sync();
}
@Override
public void hflush() throws IOException {
output.flush();
fos.getFD().sync();
}
@Override
public void write(int b) throws IOException {
output.write(b);
}
}
public class LocalInputStream extends InputStream implements Seekable, PositionedReadable, ByteBufferReadable {
private BufferedInputStream input;
private String path;
private long position = 0;
public LocalInputStream(Path path) throws IOException {
this.path = path.toString();
input = new BufferedInputStream(new FileInputStream(new RandomAccessFile(this.path, "r").getFD()), 1024*1024);
}
@Override
public int read(long l, byte[] bytes, int i, int i2) throws IOException {
throw new IOException("unsupported operation");
}
@Override
public void readFully(long l, byte[] bytes, int i, int i2) throws IOException {
throw new IOException("unsupported operation");
}
@Override
public void readFully(long l, byte[] bytes) throws IOException {
throw new IOException("unsupported operation");
}
@Override
public void seek(long l) throws IOException {
input.close();
RandomAccessFile raf = new RandomAccessFile(path, "r");
raf.seek(l);
input = new BufferedInputStream(new FileInputStream(raf.getFD()), 1024*1024);
position = l;
}
@Override
public long getPos() throws IOException {
return position;
}
@Override
public boolean seekToNewSource(long l) throws IOException {
throw new IOException("seekToNewSource not supported");
}
@Override
public int read(ByteBuffer buf) throws IOException {
buf.reset();
if(buf.hasArray()){
int read = read(buf.array(), buf.arrayOffset(), buf.capacity());
buf.limit(read);
return read;
}else{
byte[] b = new byte[buf.capacity()];
int read = read(b);
buf.put(b);
return read;
}
}
@Override
public int read(byte[] b) throws IOException {
return input.read(b);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
return input.read(b, off, len);
}
@Override
public int read() throws IOException {
byte[] b = new byte[1];
input.read(b);
position++;
return (int) b[0] & 0xFF;
}
}
}
|