Saturday, August 8, 2015

Reading/Writing a file on MapR-FS (MapR filesystem) using a java program

In this short example I will try to demonstrate a java program to Read and Write MapR filesystem.
Step 1: Add the MapR repository and MapR dependencies in the pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.mapr</groupId>
<artifactId>maprfs-example</artifactId>
<version>1.0-SNAPSHOT</version>

<repositories>
<repository>
<id>mapr-releases</id>
<url>http://repository.mapr.com/maven/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
</repositories>

<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.0.3-mapr-4.1.0</version>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.10</version>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.0.3-mapr-4.1.0</version>
<type>jar</type>
<overWrite>true</overWrite>
<outputDirectory>target</outputDirectory>
<destFileName>hadoop-core-1.0.3-mapr-4.1.0.jar</destFileName>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<finalName>${project.name}-${project.version}</finalName>
<appendAssemblyId>true</appendAssemblyId>
<descriptors>
<descriptor>src/main/assembly/assembly.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
</build>

</project>

Step 2: Java program to read and Write to the filesystem.
import java.net.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
public class ReadWriteMapRFS
{
public static void main(String args[]) throws Exception {
byte buf[] = new byte[ 65*1024];
int ac = 0;
if (args.length != 1) {
System.out.println("usage: ReadWriteMapRFS pathname");
return;
}

String dirname = args[ac++];

Configuration conf = new Configuration();
conf.set("fs.default.name", "maprfs://mapr.cluster.com:7222");
FileSystem fs = FileSystem.get(conf);

Path testDir = new Path(dirname+"/testdir");
boolean rc = fs.mkdirs( testDir);
if (!rc) {
System.out.println("unable to create directory with path " + testDir);
return;
}
System.out.println("created a new directory on path "+testDir);
Path testfile = new Path( testDir + "/testfile.txt");
System.out.println("now going to create a file inside the directory");

FSDataOutputStream fos = fs.create( testfile,true,512,(short) 1, (long)(64*1024*1024));
fos.write(buf);
fos.close();

System.out.println( "reading a recently created file : " + testfile);
FSDataInputStream istr = fs.open( testfile);
int bb = istr.readInt();
istr.close();
System.out.println( "complete read : DONE");
}
}
Post a Comment