int main(int argc, char** argv) {
GtkWidget *window=NULL;
GtkWidget *vbox=NULL;
nsCOMPtr<nsIServiceManager> servMan;
nsCOMPtr<nsILocalFile> libxul;
nsIWebNavigation *webNavigation;
int cid=0;
std::string lstStr("");
int mozSet=0;
std::string xulPath(XUL_PATH);
std::string xulrunPath(XUL_BIN);
std::string soapServer;
int maxDoc=20;
if(argc<2)
{
usage();
return 0;
}
bool isCraw=true;
int opt;
while((opt = getopt(argc, argv, "n:i:c:l:r:t:s:b:S:CG")) != -1) {
switch(opt) {
case 'C':
isCraw=true;
break;
case 'G':
isCraw=false;
break;
case 'c':
cid=atoi(optarg);
break;
case 'n':
maxDoc=atoi(optarg);
break;
case 'l':
lstStr=std::string(optarg);
break;
case 'r':
mozSet=atoi(optarg);
break;
case 's':
xulPath=std::string(optarg);
break;
case 'S':
soapServer=std::string(optarg);
break;
case 'b':
xulrunPath=std::string(optarg);
break;
default:
usage();
}
}
if(lstStr.length()==0||lstStr.length()==0)
{
usage();
}
nsDynamicFunctionLoad nsFuncs[] = {
{"XRE_InitEmbedding", (NSFuncPtr*)&XRE_InitEmbedding},
{"XRE_TermEmbedding", (NSFuncPtr*)
&XRE_TermEmbedding},
{0, 0}
};
nsresult rv;
//create native window
gtk_init (&argc, &argv);
window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
GTKEmbedGlueStartup();
GTKEmbedGlueStartupInternal();
gtk_widget_set_usize(window, 1280,1024);
vbox = gtk_vbox_new(TRUE, 0);
gtk_container_add(GTK_CONTAINER(window), vbox);
gtk_widget_show(window);
gtk_widget_show(vbox);
//init standalone env
rv = XPCOMGlueStartup(xulPath.c_str());
if (NS_FAILED(rv)) {
printf("XPCOMGlueStartup\n");
}
rv = XPCOMGlueLoadXULFunctions(nsFuncs);
if (NS_FAILED(rv)) {
printf("XPCOMGlueLoadXULFunctions\n");
}
rv = NS_NewNativeLocalFile(nsEmbedCString(xulrunPath.c_str()),
PR_FALSE, getter_AddRefs(libxul));
if (NS_FAILED(rv)) {
printf("NS_NewNativeLocalFile\n");
}
rv = XRE_InitEmbedding(libxul, 0, 0, 0, 0);
if (NS_FAILED(rv)) {
printf("XRE_InitEmbedding\n");
}
rv = NS_GetServiceManager(getter_AddRefs(servMan));
if (NS_FAILED(rv))
{
std::cout<<"Get manager Error:"<<rv<<std::endl;
}
nsCOMPtr<nsIComponentRegistrar> registrar = do_QueryInterface(servMan);
rv=registrar->AutoRegister(nsnull);
if (NS_FAILED(rv))
{
std::cout<<"Get manager Error:"<<rv<<std::endl;
}
mozilla_prefs_set_string (servMan,"general.useragent.override", "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.0.8) Gecko/2009032600 SUSE/3.0.8-1.1 Firefox/3.0.8");
nsCOMPtr<nsIStatManager> sm=do_CreateInstance(lstStr.c_str(),&rv);
if (NS_FAILED(rv))
{
std::cout<<"Get stat manager Error:"<<std::hex<<rv<<std::endl;
return 0;
}
nsCOMPtr<nsISoapInter> inter=do_CreateInstance("@nyapc.com/XPCOM/nsSoapInter;1",&rv);
if (NS_FAILED(rv))
{
std::cout<<"Init Soap Inter error\n";
return 0;
}
if(soapServer.length()>0)
inter->SetAddress(nsCString(soapServer.c_str()));
nsCOMPtr<nsISoapTask> task;
if(isCraw)
inter->GetTask(cid,maxDoc,getter_AddRefs(task));
else
inter->GetWrapTask(cid,maxDoc,getter_AddRefs(task));
sm->SetInter(inter);
std::list<gtkBrowser> bList;
if(task!=nsnull)
{
sm->SetTask(task);
PRInt32 len;
task->GetUrlCount(&len);
for(PRInt32 index=0;index<len;index++)
{
int Uid;
nsCString aurl;
task->GetUrlByIndex(index,&Uid,aurl);
gtkBrowser gb(vbox,Uid);
gb.SetListener(sm);
gb.SetRunlevel(mozSet);
gb.SetUrl(aurl);
bList.push_back(gb);
}
for(std::list<gtkBrowser>::iterator it=bList.begin();it!=bList.end();++it)
it->Start();
}else
{
sleep(60);
exit(0);
}
gtk_widget_hide(vbox);
gtk_widget_hide(window);
/**********set listener***********/
std::cout<<"Crawler ID:"<<cid<<std::endl;
gtk_main();
XRE_TermEmbedding();
XPCOMGlueShutdown();
return 0;
}
/**
* Define OS-specific constants.
*
* This function creates or uses JS object |OS.Constants| to store
* all its constants.
*/
bool DefineOSFileConstants(JSContext *cx, JSObject *global)
{
MOZ_ASSERT(gInitialized);
if (gPaths == NULL) {
// If an initialization error was ignored, we may end up with
// |gInitialized == true| but |gPaths == NULL|. We cannot
// |MOZ_ASSERT| this, as this would kill precompile_cache.js,
// so we simply return an error.
JS_ReportErrorNumber(cx, js_GetErrorMessage, NULL,
JSMSG_CANT_OPEN, "OSFileConstants", "initialization has failed");
return false;
}
JSObject *objOS;
if (!(objOS = GetOrCreateObjectProperty(cx, global, "OS"))) {
return false;
}
JSObject *objConstants;
if (!(objConstants = GetOrCreateObjectProperty(cx, objOS, "Constants"))) {
return false;
}
// Build OS.Constants.libc
JSObject *objLibc;
if (!(objLibc = GetOrCreateObjectProperty(cx, objConstants, "libc"))) {
return false;
}
if (!dom::DefineConstants(cx, objLibc, gLibcProperties)) {
return false;
}
#if defined(XP_WIN)
// Build OS.Constants.Win
JSObject *objWin;
if (!(objWin = GetOrCreateObjectProperty(cx, objConstants, "Win"))) {
return false;
}
if (!dom::DefineConstants(cx, objWin, gWinProperties)) {
return false;
}
#endif // defined(XP_WIN)
// Build OS.Constants.Sys
JSObject *objSys;
if (!(objSys = GetOrCreateObjectProperty(cx, objConstants, "Sys"))) {
return false;
}
nsCOMPtr<nsIXULRuntime> runtime = do_GetService(XULRUNTIME_SERVICE_CONTRACTID);
if (runtime) {
nsAutoCString os;
DebugOnly<nsresult> rv = runtime->GetOS(os);
MOZ_ASSERT(NS_SUCCEEDED(rv));
JSString* strVersion = JS_NewStringCopyZ(cx, os.get());
if (!strVersion) {
return false;
}
jsval valVersion = STRING_TO_JSVAL(strVersion);
if (!JS_SetProperty(cx, objSys, "Name", &valVersion)) {
return false;
}
}
// Build OS.Constants.Path
JSObject *objPath;
if (!(objPath = GetOrCreateObjectProperty(cx, objConstants, "Path"))) {
return false;
}
// Locate libxul
{
nsAutoString xulPath(gPaths->libDir);
xulPath.Append(PR_GetDirectorySeparator());
#if defined(XP_MACOSX)
// Under MacOS X, for some reason, libxul is called simply "XUL"
xulPath.Append(NS_LITERAL_STRING("XUL"));
#else
// On other platforms, libxul is a library "xul" with regular
// library prefix/suffix
xulPath.Append(NS_LITERAL_STRING(DLL_PREFIX));
xulPath.Append(NS_LITERAL_STRING("xul"));
xulPath.Append(NS_LITERAL_STRING(DLL_SUFFIX));
#endif // defined(XP_MACOSX)
if (!SetStringProperty(cx, objPath, "libxul", xulPath)) {
return false;
}
}
if (!SetStringProperty(cx, objPath, "libDir", gPaths->libDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "tmpDir", gPaths->tmpDir)) {
return false;
}
// Configure profileDir only if it is available at this stage
if (!gPaths->profileDir.IsVoid()
&& !SetStringProperty(cx, objPath, "profileDir", gPaths->profileDir)) {
return false;
}
// Configure localProfileDir only if it is available at this stage
if (!gPaths->localProfileDir.IsVoid()
&& !SetStringProperty(cx, objPath, "localProfileDir", gPaths->localProfileDir)) {
return false;
}
return true;
}
/**
* Define OS-specific constants.
*
* This function creates or uses JS object |OS.Constants| to store
* all its constants.
*/
bool DefineOSFileConstants(JSContext *cx, JS::Handle<JSObject*> global)
{
MOZ_ASSERT(gInitialized);
if (gPaths == nullptr) {
// If an initialization error was ignored, we may end up with
// |gInitialized == true| but |gPaths == nullptr|. We cannot
// |MOZ_ASSERT| this, as this would kill precompile_cache.js,
// so we simply return an error.
JS_ReportErrorNumber(cx, js_GetErrorMessage, nullptr,
JSMSG_CANT_OPEN, "OSFileConstants", "initialization has failed");
return false;
}
JS::Rooted<JSObject*> objOS(cx);
if (!(objOS = GetOrCreateObjectProperty(cx, global, "OS"))) {
return false;
}
JS::Rooted<JSObject*> objConstants(cx);
if (!(objConstants = GetOrCreateObjectProperty(cx, objOS, "Constants"))) {
return false;
}
// Build OS.Constants.libc
JS::Rooted<JSObject*> objLibc(cx);
if (!(objLibc = GetOrCreateObjectProperty(cx, objConstants, "libc"))) {
return false;
}
if (!dom::DefineConstants(cx, objLibc, gLibcProperties)) {
return false;
}
#if defined(XP_WIN)
// Build OS.Constants.Win
JS::Rooted<JSObject*> objWin(cx);
if (!(objWin = GetOrCreateObjectProperty(cx, objConstants, "Win"))) {
return false;
}
if (!dom::DefineConstants(cx, objWin, gWinProperties)) {
return false;
}
#endif // defined(XP_WIN)
// Build OS.Constants.Sys
JS::Rooted<JSObject*> objSys(cx);
if (!(objSys = GetOrCreateObjectProperty(cx, objConstants, "Sys"))) {
return false;
}
nsCOMPtr<nsIXULRuntime> runtime = do_GetService(XULRUNTIME_SERVICE_CONTRACTID);
if (runtime) {
nsAutoCString os;
DebugOnly<nsresult> rv = runtime->GetOS(os);
MOZ_ASSERT(NS_SUCCEEDED(rv));
JSString* strVersion = JS_NewStringCopyZ(cx, os.get());
if (!strVersion) {
return false;
}
JS::Rooted<JS::Value> valVersion(cx, STRING_TO_JSVAL(strVersion));
if (!JS_SetProperty(cx, objSys, "Name", valVersion)) {
return false;
}
}
#if defined(DEBUG)
JS::Rooted<JS::Value> valDebug(cx, JSVAL_TRUE);
if (!JS_SetProperty(cx, objSys, "DEBUG", valDebug)) {
return false;
}
#endif
// Build OS.Constants.Path
JS::Rooted<JSObject*> objPath(cx);
if (!(objPath = GetOrCreateObjectProperty(cx, objConstants, "Path"))) {
return false;
}
// Locate libxul
// Note that we don't actually provide the full path, only the name of the
// library, which is sufficient to link to the library using js-ctypes.
{
#if defined(XP_MACOSX)
// Under MacOS X, for some reason, libxul is called simply "XUL"
nsAutoString xulPath(NS_LITERAL_STRING("XUL"));
#else
// On other platforms, libxul is a library "xul" with regular
// library prefix/suffix
nsAutoString xulPath;
xulPath.Append(NS_LITERAL_STRING(DLL_PREFIX));
xulPath.Append(NS_LITERAL_STRING("xul"));
xulPath.Append(NS_LITERAL_STRING(DLL_SUFFIX));
#endif // defined(XP_MACOSX)
if (!SetStringProperty(cx, objPath, "libxul", xulPath)) {
return false;
}
}
if (!SetStringProperty(cx, objPath, "libDir", gPaths->libDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "tmpDir", gPaths->tmpDir)) {
return false;
}
// Configure profileDir only if it is available at this stage
if (!gPaths->profileDir.IsVoid()
&& !SetStringProperty(cx, objPath, "profileDir", gPaths->profileDir)) {
return false;
}
// Configure localProfileDir only if it is available at this stage
if (!gPaths->localProfileDir.IsVoid()
&& !SetStringProperty(cx, objPath, "localProfileDir", gPaths->localProfileDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "homeDir", gPaths->homeDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "desktopDir", gPaths->desktopDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "userApplicationDataDir", gPaths->userApplicationDataDir)) {
return false;
}
#if defined(XP_WIN)
if (!SetStringProperty(cx, objPath, "winAppDataDir", gPaths->winAppDataDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "winStartMenuProgsDir", gPaths->winStartMenuProgsDir)) {
return false;
}
#endif // defined(XP_WIN)
#if defined(XP_MACOSX)
if (!SetStringProperty(cx, objPath, "macUserLibDir", gPaths->macUserLibDir)) {
return false;
}
if (!SetStringProperty(cx, objPath, "macLocalApplicationsDir", gPaths->macLocalApplicationsDir)) {
return false;
}
#endif // defined(XP_MACOSX)
return true;
}
