Sunday, March 24, 2013

Importing the tweets from Twitter API to MongoDB

Importing the Tweets from twitter using the twitter api is very simple and straight forward.In the coming example I am importing the Sachin Tendulkar(Legendary Cricketer ) tweets to the MongoDb.Here you can find the URL of json document of Sachin tweets : http://api.twitter.com/1/statuses/user_timeline.json?screen_name=sachin_rt&include_rts=1

Java Program to import the tweets and save it the MongoDb.

package com.rajkrrsinghblogspot;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.List;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;
import com.mongodb.ServerAddress;
import com.mongodb.util.JSON;

public class GetTweets {
 
 public static void main(String[] args) throws IOException {
  try {
   MongoClient client = new MongoClient(new ServerAddress("localhost", 27017));
   
   DB database = client.getDB("tweetDb");
   DBCollection collection = database.getCollection("tweetCollection");
   
   List<DBObject> tweets = getSachinTweets();
   
   for(DBObject doc : tweets){
    collection.insert(doc);
   }
   
   System.out.println("Import done");
   
  } catch (UnknownHostException e) {
   e.printStackTrace();
  }
  
  
  
  
 }

private static List<DBObject> getSachinTweets() throws IOException {
 URL url = new URL("http://api.twitter.com/1/statuses/user_timeline.json?screen_name=sachin_rt&include_rts=1");
 InputStream is = url.openStream();
 ByteArrayOutputStream bos = new ByteArrayOutputStream();
 
 int endOfStream;
 while((endOfStream = is.read()) != -1){
  bos.write(endOfStream);
 }
 
 String tweetString = bos.toString();
 
 return (List<DBObject>) JSON.parse(tweetString);
}

}

Run the Program as Java Application but make sure that you have started the mongod (mongod --dbpath) before running the program,look out for the message import done on the console.
Now check the mongo shell