공부
[Java] Orc File
승가비
2022. 1. 10. 18:26
728x90
public static <T> Writer makeOrcWriter(String path, Class<T> clazz) throws IOException {
FileUtils.delete(path);
Configuration conf = new Configuration(false);
FileSystem fs = FileSystem.get(conf);
return OrcFile.createWriter(new Path(path),
OrcFile.writerOptions(conf)
.inspector(ObjectInspectorFactory.getReflectionObjectInspector(clazz,
ObjectInspectorFactory.ObjectInspectorOptions.JAVA))
.fileSystem(fs)
);
}
public static void delete(String path) {
new File(path).delete();
}
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hive.ql.io.orc.Writer;
import lombok.Getter;
@Getter
public class OrcFileWriter <T> implements Closeable {
private String path;
private Class<T> clazz;
private Writer writer;
private File file;
public OrcFileWriter(String path, Class<T> clazz) throws IOException {
this.path = path;
this.clazz = clazz;
writer = FileUtils.makeOrcWriter(path, clazz);
file = new File(path);
}
@Override
public void close() throws IOException {
writer.close();
}
public <T> void addRow(T o) throws IOException {
writer.addRow(o);
}
public <T> void addRow(List<T> list) throws IOException {
for (T o : list) {
writer.addRow(o);
}
}
public void delete() throws IOException {
file.delete();
close();
}
}
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReader;
import lombok.Getter;
@Getter
public class OrcFileReader {
private String path;
private File file;
private Reader reader;
public OrcFileReader(String path) throws IOException {
this.path = path;
file = new File(path);
reader = FileUtils.makeOrcReader(path);
}
public long count() {
return reader.getNumberOfRows();
}
public void print() throws IOException {
RecordReader rows = reader.rows();
while (rows.hasNext()) {
Object row = rows.next(null);
System.out.println(row);
}
rows.close();
}
}
https://orc.apache.org/docs/core-java.html
Using Core Java
Navigate the docs… Background ORC Adopters Types Indexes ACID support Building ORC Releases Spark DDL Spark Configuration Hive DDL Hive Configuration Using in MapRed Using in MapReduce Using Core Java Using Core C++ C++ Tools Java Tools Using Core Java T
orc.apache.org
https://www.javahelps.com/2020/08/read-and-write-orc-files-in-core-java.html
Read and Write ORC Files in Core Java
In this article, we will create a simple ORC writer and reader to write ORC files and to read from ORC files in core Java.
www.javahelps.com
728x90