close reader on deserialize, add a test
parent
2cfe688482
commit
52a6e3706f
|
@ -6,5 +6,8 @@
|
||||||
<classpathentry kind="lib" path="lib/default/hadoop-core.jar"/>
|
<classpathentry kind="lib" path="lib/default/hadoop-core.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/default/hive-exec.jar"/>
|
<classpathentry kind="lib" path="lib/default/hive-exec.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/default/opencsv.jar"/>
|
<classpathentry kind="lib" path="lib/default/opencsv.jar"/>
|
||||||
|
<classpathentry kind="lib" path="lib/test/junit.jar"/>
|
||||||
|
<classpathentry kind="lib" path="lib/buildtime/log4j.jar"/>
|
||||||
|
<classpathentry kind="lib" path="lib/test/commons-logging.jar"/>
|
||||||
<classpathentry kind="output" path="target/classes"/>
|
<classpathentry kind="output" path="target/classes"/>
|
||||||
</classpath>
|
</classpath>
|
||||||
|
|
1
ivy.xml
1
ivy.xml
|
@ -34,6 +34,7 @@
|
||||||
<dependency org="cobertura" name="cobertura" rev="1.9.4.1" conf="buildtime->default"/>
|
<dependency org="cobertura" name="cobertura" rev="1.9.4.1" conf="buildtime->default"/>
|
||||||
|
|
||||||
<!-- test time only dependencies -->
|
<!-- test time only dependencies -->
|
||||||
|
<dependency org="apache" name="commons-logging" rev="1.1.1" conf="test->default" />
|
||||||
<dependency org="junit" name="junit" rev="4.8.2" conf="test->default"/>
|
<dependency org="junit" name="junit" rev="4.8.2" conf="test->default"/>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</ivy-module>
|
</ivy-module>
|
||||||
|
|
|
@ -35,9 +35,9 @@ import au.com.bytecode.opencsv.CSVWriter;
|
||||||
public final class CSVSerde implements SerDe {
|
public final class CSVSerde implements SerDe {
|
||||||
|
|
||||||
private ObjectInspector inspector;
|
private ObjectInspector inspector;
|
||||||
String[] outputFields;
|
private String[] outputFields;
|
||||||
int numCols;
|
private int numCols;
|
||||||
List<String> row;
|
private List<String> row;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
|
public void initialize(final Configuration conf, final Properties tbl) throws SerDeException {
|
||||||
|
@ -102,8 +102,9 @@ public final class CSVSerde implements SerDe {
|
||||||
public Object deserialize(final Writable blob) throws SerDeException {
|
public Object deserialize(final Writable blob) throws SerDeException {
|
||||||
Text rowText = (Text) blob;
|
Text rowText = (Text) blob;
|
||||||
|
|
||||||
|
CSVReader csv = null;
|
||||||
try {
|
try {
|
||||||
final CSVReader csv = new CSVReader(new CharArrayReader(rowText.toString().toCharArray()));
|
csv = new CSVReader(new CharArrayReader(rowText.toString().toCharArray()));
|
||||||
final String[] read = csv.readNext();
|
final String[] read = csv.readNext();
|
||||||
|
|
||||||
for (int i=0; i< numCols; i++) {
|
for (int i=0; i< numCols; i++) {
|
||||||
|
@ -117,6 +118,14 @@ public final class CSVSerde implements SerDe {
|
||||||
return row;
|
return row;
|
||||||
} catch (final Exception e) {
|
} catch (final Exception e) {
|
||||||
throw new SerDeException(e);
|
throw new SerDeException(e);
|
||||||
|
} finally {
|
||||||
|
if (csv != null) {
|
||||||
|
try {
|
||||||
|
csv.close();
|
||||||
|
} catch (final Exception e) {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
package com.bizo.hive.serde.csv;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hive.serde.Constants;
|
||||||
|
import org.apache.hadoop.io.Text;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
|
||||||
|
|
||||||
|
public final class CSVSerdeTest {
|
||||||
|
private final CSVSerde csv = new CSVSerde();
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() throws Exception {
|
||||||
|
final Properties props = new Properties();
|
||||||
|
props.put(Constants.LIST_COLUMNS, "a,b,c");
|
||||||
|
props.put(Constants.LIST_COLUMN_TYPES, "string,string,string");
|
||||||
|
|
||||||
|
csv.initialize(null, props);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDeserialize() throws Exception {
|
||||||
|
final Text in = new Text("hello,\"yes, okay\",1");
|
||||||
|
|
||||||
|
final List<String> row = (List<String>) csv.deserialize(in);
|
||||||
|
|
||||||
|
assertEquals("hello", row.get(0));
|
||||||
|
assertEquals("yes, okay", row.get(1));
|
||||||
|
assertEquals("1", row.get(2));
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue