misc/bashloader/bashloader.cpp

/*
 *
 * bashloader
 *
 * for use with following table
 *
 * CREATE TABLE bash (
 *     id int auto_increment primary key,
 *     bashid int,
 *     rating int,
 *     quote text,
 *     KEY bash_rating(rating)
 * );
 */

#include <iostream>
#include <sstream>
#include <string>

#include <stdlib.h>

#include "readUrl.h"

using namespace std;

void cleanString(string* str, string search, string replace = "")
{
        unsigned int pos = 0;
        while ( (pos = str->find(search,pos)) != string::npos)
        {
                str->replace(pos, search.size(), replace);
                pos += replace.size();
        }
}

void parseQuote(std::string str)
{
        int pos, end_pos;
        string quote, str_number,str_rating;
//      cout << "-----------------------------------------------"<< endl;
        pos = str.find("#");
        pos++;
        end_pos = str.find("<", pos);
        str_number = str.substr(pos, end_pos-pos);

        pos = str.find("(", end_pos);
        pos++;
        end_pos  = str.find(")", pos);
        str_rating = str.substr(pos, end_pos-pos);
        pos = str.find("<p");

        pos += 14;
        quote = str.substr(pos);

        cleanString(&quote, "\r");
        cleanString(&quote, "&amp;", "&");
        cleanString(&quote, "<br />");
        cleanString(&quote, "&lt;", "<");
        cleanString(&quote, "&gt;", ">");
        cleanString(&quote, "&nbsp;", " ");
        cleanString(&quote, "\\", "\\\\");
        cleanString(&quote, "&quot;", "\"");
        cleanString(&quote, "'", "\\'");

        cout << "INSERT INTO bash (bashid, rating, quote) values (" << str_number << "," << str_rating << ",'" << quote << "');" << endl;
}

void parseDocument(int i)
{
        ostringstream url;
        url << "http://bash.org/?browse&p=" << i;
        string document = readUrl( url.str() );
                
        unsigned int pos=0, end_pos;
        while(1)
        {
                pos = document.find("<p class=\"quote\">",pos);
                if (pos == string::npos)
                                break;
                pos += 10;
                end_pos = document.find("</p>", pos);

                end_pos = document.find("</p>", end_pos+4);
                
                parseQuote(document.substr(pos, end_pos-pos) );
        }
        
}

int main()
{
        unsigned pos, end_pos, max;
        string document = readUrl("http://bash.org/?browse");

        if (document == "")
                exit(1);

        pos = document.rfind("<option value=");
        pos += 15;
        end_pos = document.find("\"", pos+1);
        max = atoi(  document.substr(pos, end_pos-pos).c_str() );

        cout << "CREATE TABLE IF NOT EXISTS bash (" << endl;
        cout << " id int auto_increment primary key," << endl;
        cout << " bashid int," << endl;
        cout << " rating int," << endl;
        cout << " quote text," << endl;
        cout << " KEY bash_rating(rating)" << endl;
        cout << " );" << endl << endl;

        cout << "TRUNCATE TABLE bash;" << endl;
        //parseDocument(2);
        for (unsigned int i=1; i<= max; i++)
        {
                cerr << "Parsing #" << i << "/" << max << endl;
                parseDocument(i);
                usleep(50*1000); //be nice to the webserver

        }
}
1	torben	547	/*
2			*
3			* bashloader
4			*
5			* for use with following table
6			*
7			* CREATE TABLE bash (
8			* id int auto_increment primary key,
9			* bashid int,
10			* rating int,
11			* quote text,
12			* KEY bash_rating(rating)
13			* );
14			*/
15
16			#include <iostream>
17			#include <sstream>
18			#include <string>
19
20			#include <stdlib.h>
21
22			#include "readUrl.h"
23
24			using namespace std;
25
26			void cleanString(string* str, string search, string replace = "")
27			{
28			unsigned int pos = 0;
29			while ( (pos = str->find(search,pos)) != string::npos)
30			{
31			str->replace(pos, search.size(), replace);
32			pos += replace.size();
33			}
34			}
35
36			void parseQuote(std::string str)
37			{
38			int pos, end_pos;
39			string quote, str_number,str_rating;
40			// cout << "-----------------------------------------------"<< endl;
41			pos = str.find("#");
42			pos++;
43			end_pos = str.find("<", pos);
44			str_number = str.substr(pos, end_pos-pos);
45
46			pos = str.find("(", end_pos);
47			pos++;
48			end_pos = str.find(")", pos);
49			str_rating = str.substr(pos, end_pos-pos);
50			pos = str.find("<p");
51
52			pos += 14;
53			quote = str.substr(pos);
54
55			cleanString(&quote, "\r");
56			cleanString(&quote, "&", "&");
57			cleanString(&quote, "<br />");
58			cleanString(&quote, "<", "<");
59			cleanString(&quote, ">", ">");
60			cleanString(&quote, " ", " ");
61			cleanString(&quote, "\\", "\\\\");
62			cleanString(&quote, """, "\"");
63			cleanString(&quote, "'", "\\'");
64
65			cout << "INSERT INTO bash (bashid, rating, quote) values (" << str_number << "," << str_rating << ",'" << quote << "');" << endl;
66			}
67
68			void parseDocument(int i)
69			{
70			ostringstream url;
71			url << "http://bash.org/?browse&p=" << i;
72			string document = readUrl( url.str() );
73
74			unsigned int pos=0, end_pos;
75			while(1)
76			{
77			pos = document.find("<p class=\"quote\">",pos);
78			if (pos == string::npos)
79			break;
80			pos += 10;
81			end_pos = document.find("</p>", pos);
82
83			end_pos = document.find("</p>", end_pos+4);
84
85			parseQuote(document.substr(pos, end_pos-pos) );
86			}
87
88			}
89
90			int main()
91			{
92			unsigned pos, end_pos, max;
93			string document = readUrl("http://bash.org/?browse");
94
95			if (document == "")
96			exit(1);
97
98			pos = document.rfind("<option value=");
99			pos += 15;
100			end_pos = document.find("\"", pos+1);
101			max = atoi( document.substr(pos, end_pos-pos).c_str() );
102
103			cout << "CREATE TABLE IF NOT EXISTS bash (" << endl;
104			cout << " id int auto_increment primary key," << endl;
105			cout << " bashid int," << endl;
106			cout << " rating int," << endl;
107			cout << " quote text," << endl;
108			cout << " KEY bash_rating(rating)" << endl;
109			cout << " );" << endl << endl;
110
111			cout << "TRUNCATE TABLE bash;" << endl;
112			//parseDocument(2);
113			for (unsigned int i=1; i<= max; i++)
114			{
115			cerr << "Parsing #" << i << "/" << max << endl;
116			parseDocument(i);
117			usleep(50*1000); //be nice to the webserver
118
119			}
120			}