/*
 *
 * bashloader
 *
 * for use with following table
 *
 * CREATE TABLE bash (
 *     id int auto_increment primary key,
 *     bashid int,
 *     rating int,
 *     quote text,
 *     KEY bash_rating(rating)
 * );
 */

#include <iostream>
#include <sstream>
#include <string>

#include <stdlib.h>

#include "readUrl.h"

using namespace std;

void cleanString(string* str, string search, string replace = "")
{
	unsigned int pos = 0;
	while ( (pos = str->find(search,pos)) != string::npos)
	{
		str->replace(pos, search.size(), replace);
		pos += replace.size();
	}
}

void parseQuote(std::string str)
{
	int pos, end_pos;
	string quote, str_number,str_rating;
//	cout << "-----------------------------------------------"<< endl;
	pos = str.find("#");
	pos++;
	end_pos = str.find("<", pos);
	str_number = str.substr(pos, end_pos-pos);

	pos = str.find("(", end_pos);
	pos++;
	end_pos  = str.find(")", pos);
	str_rating = str.substr(pos, end_pos-pos);
	pos = str.find("<p");

	pos += 14;
	quote = str.substr(pos);

	cleanString(&quote, "\r");
	cleanString(&quote, "&amp;", "&");
	cleanString(&quote, "<br />");
	cleanString(&quote, "&lt;", "<");
	cleanString(&quote, "&gt;", ">");
	cleanString(&quote, "&nbsp;", " ");
	cleanString(&quote, "\\", "\\\\");
	cleanString(&quote, "&quot;", "\"");
	cleanString(&quote, "'", "\\'");

	cout << "INSERT INTO bash (bashid, rating, quote) values (" << str_number << "," << str_rating << ",'" << quote << "');" << endl;
}

void parseDocument(int i)
{
	ostringstream url;
	url << "http://bash.org/?browse&p=" << i;
	string document = readUrl( url.str() );
		
	unsigned int pos=0, end_pos;
	while(1)
	{
		pos = document.find("<p class=\"quote\">",pos);
		if (pos == string::npos)
				break;
		pos += 10;
		end_pos = document.find("</p>", pos);

		end_pos = document.find("</p>", end_pos+4);
		
		parseQuote(document.substr(pos, end_pos-pos) );
	}
	
}

int main()
{
	unsigned pos, end_pos, max;
	string document = readUrl("http://bash.org/?browse");

	if (document == "")
		exit(1);

	pos = document.rfind("<option value=");
	pos += 15;
	end_pos = document.find("\"", pos+1);
	max = atoi(  document.substr(pos, end_pos-pos).c_str() );

	cout << "CREATE TABLE IF NOT EXISTS bash (" << endl;
	cout << " id int auto_increment primary key," << endl;
	cout << " bashid int," << endl;
	cout << " rating int," << endl;
	cout << " quote text," << endl;
	cout << " KEY bash_rating(rating)" << endl;
	cout << " );" << endl << endl;

	cout << "TRUNCATE TABLE bash;" << endl;
	//parseDocument(2);
	for (unsigned int i=1; i<= max; i++)
	{
		cerr << "Parsing #" << i << "/" << max << endl;
		parseDocument(i);
		usleep(50*1000); //be nice to the webserver

	}
}