Location: PHPKode > projects > Astanda Open Directory Project > adp-1.3b/admin/functions.php
<?php
//-------------------------------------------------//
//                                                 //
//        ASTANDA DIRECTORY PROJECT  [ADP]         //
//                  Version 1.3b                   //
//               License: SHAREWARE                //
//             http://www.astanda.com              //
//      © Copyright 2005, 2006, Pavel Golovko      //
//                                                 //
//-------------------------------------------------//
//                                                 //
// Any attempt to remove or alter "Powered by ADP" //
// recip link will automatically wave your rights  //
// to any use of ADP software!                     //
//                                                 //
// Reverse engineering and redistribution of       //
// ADP code or any part of its code is strictly    //
// prohibited!                                     //
//                                                 //
//-------------------------------------------------//
//                                                 //
// ADP is distributed in the hope that it will be  //
// useful, but WITHOUT ANY WARRANTY; without even  //
// the implied warranty of MERCHANTABILITY or      //
// FITNESS FOR A PARTICULAR PURPOSE.               //
//                                                 //
//-------------------------------------------------//

function cfg($v){ $query = "SELECT `$v` FROM `config` LIMIT 1"; $result = mysql_query($query); echo mysql_error(); if(mysql_num_rows($result)){ while($cfg = mysql_fetch_array($result)){ return $cfg[$v]; } } } function menu($type){ switch ($type) { case "categories": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Categories:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=categories&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=categories&action=add">Add New</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "links": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Links:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=add">New</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=approve">Approve</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=move">Move</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=dupes">Dupes</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=recip">Check Recip</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=links&action=gpr">Update Google PR</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "pages": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Pages:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=pages&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=pages&action=date">Index Dates</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=pages&action=reindex">Reindex All</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=pages&action=recip">Check Recip</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=pages&action=gpr">Update Google PR</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "import": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Import:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=import&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=import&action=dmoz">DMOZ</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "banned": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Banned:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=banned&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=banned&action=keywords">Keywords</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "config": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Configuration:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=config&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=config&action=spider">Spider</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=config&action=admin">Admin Area</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=config&action=database">Database</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=config&action=password&pwd=setup">Admin Password</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "build": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Build:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=build&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=build&action=gsm">Google SiteMaps</a>(<a href="index.php?adp=build&action=gsm1">Delete</a>)</font></td> <td nowrap><font size="2">· <a href="index.php?adp=build&action=pages">Static Pages</a>(<a href="index.php?adp=build&action=pages1">Delete</a>)</font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; case "log": ?> <table width="100%" border=0 cellpadding=0 cellspacing=0><tr><td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td></tr></table> <table border="0" width="95%" cellpadding="2"> <tr> <td nowrap><font size="2"><b>Log:</b></font></td> <td nowrap><font size="2">· <a href="index.php?adp=log&action=main">Main</a></font></td> <td nowrap><font size="2">· <a href="index.php?adp=log&action=search">Search Log</a></font></td> <td width="90%">&nbsp;</td> </tr> </table> <? break; } ?> </center> </div> </td> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> </tr> <tr> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> </tr> </table> </center> </div> &nbsp; <div align="center"> <center> <table border="0" width="85%" cellspacing="0" cellpadding="0"> <tr> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> </tr> <tr> <td bgcolor="#008000"><img border="0" src="images/pixel.png" width="1" height="1"></td> <td width="100%" bgcolor="#FFFFFF"> <div align="center"> <center> <table border="0" width="99%" cellpadding="1"> <tr> <td><font size="2"> <? } function list_cats($parent, $lev, $exclude, $selected) { $print = ""; if ($lev == 0) { print "\n"; } $space = ""; for ($x = 0; $x < $lev; $x++) { $space .= "&nbsp;&nbsp;&nbsp;·&nbsp;"; } $result = mysql_query("SELECT * FROM `categories` WHERE `parent` = '$parent' ORDER BY `title` ASC"); echo mysql_error(); if (mysql_num_rows($result) <> '') while ($row = mysql_fetch_array($result)) { $result_num = mysql_query("SELECT * FROM `links` WHERE `category_id` = '".$row['id']."'"); echo mysql_error(); $links_num = mysql_num_rows($result_num); $id = $row['id']; $title = $row['title']; $sel = ""; if($id == $selected){ $sel = " selected "; } if($id != $exclude){ print "<option $sel value=\"$id\">$space$title ($links_num)</option>"; } $print = list_cats($id, $lev + 1, $exclude, $selected); } return $print; } function check_cats($parent, $lev) { if ($lev == 0) { print "\n"; } $space = ""; for ($x = 0; $x < $lev; $x++) { $space .= "&nbsp;&nbsp;&nbsp;·&nbsp;"; } $query = "SELECT * FROM `categories` WHERE `parent` = '$parent' ORDER BY `title` ASC"; $result = mysql_query($query); echo mysql_error(); if (mysql_num_rows($result) <> '') while ($row = mysql_fetch_array($result)) { $query_num = "SELECT * FROM `links` WHERE `category_id` = '".$row['id']."'"; $result_num = mysql_query($query_num); echo mysql_error(); $links_num = mysql_num_rows($result_num); $id = $row['id']; $title = $row['title']; if($row['path'] == "") { $err = "<font size=1 color=red>ERROR: NO CAT. NAME</font>"; $error = "1"; } print "<tr bgcolor=\"#FFFFFF\" onMouseOver=\"this.bgColor='#F4F4F4';\" onMouseOut=\"this.bgColor='#FFFFFF';\"><form action=\"index.php\" method=\"POST\"><td><input type=\"hidden\" name=\"adp\" value=\"categories\"><input type=\"hidden\" name=\"action\" value=\"edit\"><input type=\"hidden\" name=\"category_id\" value=\"$id\"><font size=1>$space$title ($links_num)</td><td>$err</td><td><input type=\"image\" src=\"images/edit.png\"></td></form></tr>"; $print = check_cats($id, $lev + 1); } return $print; } function where_ami($id){ if($id <> "0"){ $query = "SELECT `parent`, `title` FROM `categories` WHERE `id` = '$id' LIMIT 1"; $result = mysql_query($query); echo mysql_error(); if (mysql_num_rows($result) <> ''){ while ($row = mysql_fetch_array($result)) { $again = where_ami($row['0']); $title = $row['1']; print " / <a href=\"index.php?adp=categories&action=edit&category_id=$id\">$title</a>"; } } return $again; } } function main_cats($parent,$lev,$orderby,$how) { $print = ""; if($lev == 0){ print "\n"; } if(!$how){ $how = "ASC"; } if($orderby <> "date"){ $orderby = "title"; } $space = ""; for($x=0;$x<$lev;$x++){ $space .= "&nbsp;&nbsp;&nbsp;·&nbsp;"; } $result = mysql_query("SELECT * FROM `categories` WHERE `parent` = '$parent' ORDER BY `$orderby` $how"); echo mysql_error(); if(mysql_num_rows($result)) while($row = mysql_fetch_array($result)){ $result_num = mysql_query("SELECT `id` FROM `links` WHERE `category_id` = '".$row['id']."'"); echo mysql_error(); $links_num = mysql_num_rows($result_num); $id = $row['id']; $title = $row['title']; print "<tr bgcolor=\"#F4F4F4\" onMouseOver=\"this.bgColor='#FFFFFF';\" onMouseOut=\"this.bgColor='#F4F4F4';\">"; print "<td><a href=\"index.php?adp=categories&action=edit&category_id=$id\"><img border=0 src=\"images/edit.png\"></a></td>"; print "<td width=\"98%\"><font size=1>$space$title (<a href=\"index.php?adp=links&action=category&cat_id=$id\">$links_num</a>)</td>"; print "<td><a target=\"_blank\" href=\"spider.php?cat_id=$id\"><img border=\"0\" src=\"images/spider.png\"></a></td>"; print "<td><font size=1>".$row['date']."</td>"; print "<td><a href=\"index.php?adp=categories&action=delete&cat_id=$id\"><img border=0 src=\"images/delete.png\"></a></td></tr>"; $print = main_cats($id,$lev + 1,$orderby,$how); } return $print; } function total_links($cat_id){ $out = ""; $all_cats = get_cats($cat_id); for($a=0;$a<=count($all_cats)-1;$a++){ $out .= $all_cats[$a].","; } $result = mysql_query("SELECT COUNT(`id`) FROM `links` WHERE `visible` = '1' AND `category_id` IN (".trim($out,',').")"); echo mysql_error(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ return $row['0']; } } } function get_cats($parent) { $result = mysql_query("SELECT `id` FROM `categories` WHERE `parent` = '$parent'"); echo mysql_error(); $arr[] = $parent; if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $arr = add_arrays($arr,get_cats($row['id'])); } } return $arr; } function add_arrays($arr1, $arr2) { foreach ($arr2 as $elem) { $arr1[] = $elem; } return $arr1; } function a($a, &$smarty){ return preg_replace(l('7e3c5c2f626f64793e7e69'),l('3c63656e7465723e3c666f6e742073697a653d2231223e506f7765726564206279203c61207461726765743d225f626c616e6b2220687265663d22687474703a2f2f7777772e617374616e64612e636f6d2f6164702f223e4144503c2f613e3c2f666f6e743e3c2f63656e7465723e3c2f626f64793e'),$a); } function fetchURL($url){ if(!is_array($url)){ $url=trim($url); if(substr($url,0,7)!="http://"){ $url="http://".$url; } $url = splitURL($url); $fp = @ fsockopen($url['host'], $url['port'], $errno, $errstr, cfg("spider_ConnectionTimeout")); if($fp==false){ $html = "$errstr ($errno)"; } else{ $out = "GET ".$url['path'].$url['file'].$url['query']." HTTP/1.0\r\n"; $out .= "Host: ".$url['host']."\r\n"; $out .= "Referer: ".$url."\r\n"; $out .= "User-Agent: ".cfg("spider_UserAgent")."\r\n"; $out .= "Content-Type: text/html\r\n"; $out .= "Connection: Close\r\n\r\n"; fwrite($fp, $out); $status=socket_get_status($fp); $html = ""; while($status["timed_out"]!=true && !feof($fp)){ stream_set_timeout($fp, cfg("spider_StreamTimeout")); $html .= fgets($fp,1024); } fclose($fp); } return $html; } } function splitURL(&$url) { preg_match("/^.{1,10}:\/\/[^\/\?]{1,}(\/|$|\?){1}/", $url, $match); if (isset($match[0])) { $match[0]=preg_replace("/^.{0,}:\/\//","",$match[0]); $match[0]=str_replace("?","",$match[0]); $match[0]=str_replace("/","",$match[0]); $url_parts["host"]=$match[0]; } else $url_parts["host"]=""; unset($match); @preg_match("/".str_replace(".", '\.', $url_parts["host"])."\/{1}[^\?]{0,}\//", $url, $match); if(isset($match[0])){ $match[0]=str_replace($url_parts["host"], "", $match[0]); $url_parts["path"]=trim($match[0]); } else{ $url_parts["path"]="/"; } preg_match("/[^\/]{1}\/[^\/\?]{1,}(\?|$)/", $url, $match); if(isset($match[0])){ $match[0]=str_replace("/", "", $match[0]); $match[0]=str_replace("?", "", $match[0]); $match[0]=substr($match[0], 1); $file=$match[0]; } else{ $file = ""; } preg_match("/\?.{0,}/", $url, $match); if(isset($match[0])){ $query=$match[0]; } else{ $query = ""; } $parts=@explode(".", $url_parts["host"]); if(count($parts)<=2){ $domain=$url_parts["host"]; } else{ $pos=strpos($url_parts["host"], "."); $domain=substr($url_parts["host"], $pos+1); } preg_match("/^.{0,10}\/\//", $url, $match); if(isset($match[0])){ $protocol=$match[0]; } else{ $protocol = ""; } $r["host"]=$url_parts["host"]; $r["port"] = 80; $r["path"]=$url_parts["path"]; $r["file"]=$file; $r["query"]=$query; $r["domain"]=$domain; $r["protocol"]=$protocol; return $r; } class ADPcrawler { var $url_to_crawl="http://www.foo.com"; var $port=80; var $socket_mean_timeout=10; var $socket_read_timeout=2; var $traffic_limit_all=0; var $traffic_limit_complete_page=true; var $page_limit_all=0; var $page_limit_count_ct_only=true; var $pagesize_limit=0; var $general_follow_mode=2; var $follow_redirects_till_content=true; var $follow_content_type=array(); var $not_follow_matches=array(); var $follow_matches=array(); var $follow_redirects=true; var $handle_cookies=true; var $base_file; var $base_path; var $base_host; var $base_domain; var $traffic_all; var $urls_to_crawl=array(); var $referers_to_urls_to_crawl=array(); var $cookies=array(); var $content_found=false; var $status_return=array(); function crawler() { } function &crawl(&$protocol, &$host, &$path, &$file, &$query, &$referer_url) { $s = @fsockopen ($host, $this->port, $e, $t, $this->socket_mean_timeout); if ($s==false) { $error_string=$t; $error_code=$e; } else { $header_found=false; $headerlines_to_send[] = "GET ".$path.$file.$query." HTTP/1.0\r\n"; $headerlines_to_send[] = "HOST: ".$host."\r\n"; if ($referer_url!="") { $headerlines_to_send[] = "Referer: $referer_url\r\n"; } @reset($this->cookies[$host]); while(list($cookie_var)=@each($this->cookies[$host])) { if (isset($cookie_string)) $cookie_string.="; ".$cookie_var."=".$this->cookies[$host][$cookie_var]; else $cookie_string="; ".$cookie_var."=".$this->cookies[$host][$cookie_var]; } if (isset($cookie_string)) { $cookie_string = substr($cookie_string, 2); $headerlines_to_send[] = "Cookie: ".$cookie_string."\r\n"; } $headerlines_to_send[] = "User-Agent: ".cfg("spider_UserAgent")."\r\n"; $headerlines_to_send[] = "Connection: close\r\n"; $headerlines_to_send[] = "\r\n"; for ($x=0; $x<count($headerlines_to_send); $x++) { fputs($s, $headerlines_to_send[$x]); if (isset($header_send)) $header_send .= $headerlines_to_send[$x]; else $header_send = $headerlines_to_send[$x]; } $status=socket_get_status($s); $source_read = ""; while ($status["timed_out"]!=true && !feof($s) && !isset($stop)) { socket_set_timeout($s, $this->socket_read_timeout); $source_read.=fgets($s,1024); if ($this->traffic_limit_complete_page==false && $this->traffic_limit_all>0) { if (strlen($source_read) + $this->traffic_all > $this->traffic_limit_all) { $stop=true; $received_completly=false; } } if ($header_found==true && (strlen($source_read) > $this->pagesize_limit) && $this->pagesize_limit > 0) { $stop=true; $received_completly=false; } if ($header_found==false && substr($source_read,-4,4)=="\r\n\r\n") { $header=substr($source_read, 0, strlen($source_read)-2); $source_read = ""; $header_found=true; $http_status_code=ADPcrawlerUtils::getHTTPStatusCode($header); $follow=ADPcrawlerUtils::decideFollow($header, $this->follow_content_type); if ($follow==false) { $stop=true; } else { $received_completly=true; } if ($this->handle_cookies==true) { ADPcrawlerUtils::getCookieData($header, $this->cookies, $host); } } $status=socket_get_status($s); if ($status["timed_out"]==true) { $error_code=1000; $error_string="socketstream timed out"; $stop=true; $received_completly=false; } if ($status["eof"]==true) { $stop=true; } } fclose($s); } unset($page_data); if (isset($error_code)) $page_data["error_code"]=$error_code; else $page_data["error_code"]=false; if (isset($error_string)) $page_data["error_string"]=$error_string; else $page_data["error_string"]=false; if (isset($follow)) $page_data["received"]=&$follow; else $page_data["received"]=false; if (isset($received_completly)) $page_data["received_completly"]=&$received_completly; else $page_data["received_completly"]=false; $page_data["bytes_received"]=strlen($source_read); if (isset($header)) $page_data["header"]=&$header; else $page_data["header"]=false; if (isset($http_status_code)) $page_data["http_status_code"]=&$http_status_code; else $page_data["http_status_code"]=false; $page_data["source"]=&$source_read; $page_data["header_send"]=&$header_send; $page_data["protocol"]=$protocol; $page_data["host"]=$host; $page_data["path"]=$path; $page_data["file"]=$file; $page_data["query"]=$query; $page_data["url"]=$protocol.$host.$path.$file.$query; return $page_data; } function go() { $url_parts=ADPcrawlerUtils::splitURL($this->url_to_crawl); $this->base_path=$url_parts["path"]; $this->base_host=$url_parts["host"]; $this->base_domain=$url_parts["domain"]; $this->urls_to_crawl[]=$url_parts["protocol"].$url_parts["host"].$url_parts["path"].$url_parts["file"].$url_parts["query"]; $this->referers_to_urls_to_crawl[]=""; $links_followed=0; $files_received=0; while (list($key)=@each($this->urls_to_crawl)) { $parts_of_actual_url=ADPcrawlerUtils::splitURL($this->urls_to_crawl[$key]); unset($page_data); $page_data=&$this->crawl($parts_of_actual_url["protocol"], $parts_of_actual_url["host"], $parts_of_actual_url["path"], $parts_of_actual_url["file"], $parts_of_actual_url["query"], $this->referers_to_urls_to_crawl[$key]); $links_followed++; if ($page_data["http_status_code"]==200) $content_found=true; if ($page_data["received"]==true) { $files_received++; } $this->traffic_all = $this->traffic_all+strlen($page_data["header"])+strlen($page_data["source"]); if ($this->traffic_all > $this->traffic_limit_all && $this->traffic_limit_all!=0) { $stop_crawling=true; $page_data["traffic_limit_reached"]=true; } else $page_data["traffic_limit_reached"]=false; if ($this->page_limit_all > 0) { if ($this->page_limit_count_ct_only==true && $files_received>=$this->page_limit_all) { $stop_crawling=true; $page_data["file_limit_reached"]=true; } elseif ($this->page_limit_count_ct_only==false && $links_followed>=$this->page_limit_all) { $stop_crawling=true; $page_data["file_limit_reached"]=true; } } else $page_data["file_limit_reached"]=false; $page_data["referer_url"]=&$this->referers_to_urls_to_crawl[$key]; $this->handlePageData($page_data); if (isset($stop_crawling)) { break; } unset($links_found); $links_found=ADPcrawlerUtils::find_links($page_data["source"]); $base_url=ADPcrawlerUtils::getBasePathFromTag($page_data["source"]); $redirect=ADPcrawlerUtils::getRedirectLocation($page_data["header"]); if ($redirect && $this->follow_redirects==true) { $rd[0]=$redirect; $links_found=@array_merge($rd, $links_found); } if ($base_url=="") $actual_url=&$this->urls_to_crawl[$key]; else $actual_url=$base_url; ADPcrawlerUtils::buildURLs($links_found, $actual_url); if ($this->general_follow_mode==2 || $this->general_follow_mode==3) { ADPcrawlerUtils::removeURLsToOtherHosts($links_found, $this->urls_to_crawl[$key]); } if ($this->general_follow_mode==1) { ADPcrawlerUtils::removeURLsToOtherDomains($links_found, $this->urls_to_crawl[$key]); } if ($this->general_follow_mode==3) { ADPcrawlerUtils::removePathUpLinks($links_found, $this->url_to_crawl); } if (count($this->not_follow_matches)>0) { ADPcrawlerUtils::removeMatchingLinks($links_found, $this->not_follow_matches); } if (count($this->follow_matches)>0) { $links_found=&ADPcrawlerUtils::removeNotMatchingLinks($links_found, $this->follow_matches); } ADPcrawlerUtils::addToArray($links_found, $this->urls_to_crawl, $this->urls_to_crawl[$key], $this->referers_to_urls_to_crawl); if ($content_found==false && $rd[0]!="" && $this->follow_redirects_till_content==true) { ADPcrawlerUtils::addToArray($rd, $this->urls_to_crawl, $this->urls_to_crawl[$key], $this->referers_to_urls_to_crawl); } } $this->status_return["links_followed"]=$links_followed; $this->status_return["files_received"]=$files_received; $this->status_return["bytes_received"]=$this->traffic_all; $this->status_return["traffic_limit_reached"]=$page_data["traffic_limit_reached"]; $this->status_return["file_limit_reached"]=$page_data["file_limit_reached"]; if ($stop_crawling==true) { $this->status_return["limit_reached"]=true; } else { $this->status_return["limit_reached"]=false; } } function handlePageData(&$page_data) { } function setURL($url) { $url=trim($url); if (substr($url,0,7)!="http://") { $url="http://".$url; } $this->url_to_crawl=$url; } function setFollowMode($mode) { $this->general_follow_mode=$mode; } function setPageLimit($limit, $mode=false) { $this->page_limit_count_ct_only=$mode; $this->page_limit_all=$limit; } function setTrafficLimit($limit, $mode=true) { $this->traffic_limit_all=$limit; $this->traffic_limit_complete_page=$mode; } function setContentSizeLimit($limit) { $this->pagesize_limit=$limit; } function addReceiveContentType($content_type) { $this->follow_content_type[]=trim(strtolower($content_type)); } function addNonFollowMatch ($preg_follow) { $this->not_follow_matches[]=trim($preg_follow); } function addFollowMatch ($preg_follow) { $this->follow_matches[]=trim($preg_follow); } function setFollowRedirects ($mode) { $this->follow_redirects=$mode; } function setFollowRedirectsTillContent($mode) { $this->follow_redirects_till_content=$mode; } function setCookieHandling ($mode) { $this->handle_cookies=$mode; } function setConnectionTimeout($timeout) { $this->socket_mean_timeout=$timeout; } function setStreamTimeout($timeout) { $this->socket_read_timeout=$timeout; } function getReport() { return $this->status_return; } } class ADPcrawlerUtils { function splitURL(&$url) { preg_match("/^.{1,10}:\/\/[^\/\?]{1,}(\/|$|\?){1}/", $url, $match); if (isset($match[0])) { $match[0]=preg_replace("/^.{0,}:\/\//","",$match[0]); $match[0]=str_replace("?","",$match[0]); $match[0]=str_replace("/","",$match[0]); $url_parts["host"]=$match[0]; } else $url_parts["host"]=""; unset($match); @preg_match("/".str_replace(".", '\.', $url_parts["host"])."\/{1}[^\?]{0,}\//", $url, $match); if (isset($match[0])) { $match[0]=str_replace($url_parts["host"], "", $match[0]); $url_parts["path"]=trim($match[0]); } else $url_parts["path"]="/"; preg_match("/[^\/]{1}\/[^\/\?]{1,}(\?|$)/", $url, $match); if (isset($match[0])) { $match[0]=str_replace("/", "", $match[0]); $match[0]=str_replace("?", "", $match[0]); $match[0]=substr($match[0], 1); $file=$match[0]; } else $file = ""; preg_match("/\?.{0,}/", $url, $match); if (isset($match[0])) $query=$match[0]; else $query = ""; $parts=@explode(".", $url_parts["host"]); if (count($parts)<=2) { $domain=$url_parts["host"]; } else { $pos=strpos($url_parts["host"], "."); $domain=substr($url_parts["host"], $pos+1); } preg_match("/^.{0,10}\/\//", $url, $match); if (isset($match[0])) $protocol=$match[0]; else $protocol = ""; $r["host"]=$url_parts["host"]; $r["path"]=$url_parts["path"]; $r["file"]=$file; $r["query"]=$query; $r["domain"]=$domain; $r["protocol"]=$protocol; return $r; } function find_links(&$source) { $all_links=array(); preg_match_all("/((?i)href|src)[ ]{0,}=[ ]{0,}(\"|'){0,1}[^\"'><\n ]{0,}(\"|'|>|<|\n| )/",$source,$regs); $all_links=array_merge($all_links,$regs[0]); preg_match_all("/((?i)location)[ ]{0,}=[ ]{0,}(\"|'){1}[^\"'><\n ]{0,}(\"|')/",$source,$regs); $all_links=array_merge($all_links,$regs[0]); preg_match_all("/((?i)open)[ ]{0,}\([ ]{0,}(\"|'){1}[^\"'><\n ]{0,}(\"|')/",$source,$regs); $all_links=array_merge($all_links,$regs[0]); preg_match_all("/((?i)url)[ ]{0,}=[ ]{0,}[^\"'><\n ]{0,}(\"|>| |')/",$source,$regs); $all_links=array_merge($all_links,$regs[0]); for ($x=0;$x<count($all_links);$x++) { $all_links[$x]=str_replace("\"","",$all_links[$x]); $all_links[$x]=str_replace("'","",$all_links[$x]); $all_links[$x]=str_replace(">","",$all_links[$x]); $all_links[$x]=str_replace("<","",$all_links[$x]); $all_links[$x]=preg_replace("/((?i)href|src|location|url)[ ]{0,}=/","",$all_links[$x]); $all_links[$x]=preg_replace("/((?i)open)[ ]{0,}\(/","",$all_links[$x]); $all_links[$x]=preg_replace("/((?i)url)[ ]{0,}=/","",$all_links[$x]); $all_links[$x]=trim($all_links[$x]); } return($all_links); } function buildURLs(&$links, $actual_url) { $url_parts_actual=ADPcrawlerutils::splitURL($actual_url); $entities= array ("'&(quot|#34);'i", "'&(amp|#38);'i", "'&(lt|#60);'i", "'&(gt|#62);'i", "'&(nbsp|#160);'i", "'&(iexcl|#161);'i", "'&(cent|#162);'i", "'&(pound|#163);'i", "'&(copy|#169);'i"); $replace=array ("\"", "&", "<", ">", " ", chr(161), chr(162), chr(163), chr(169)); @reset($links); while (list($x)=@each($links)) { $url_parts_link=ADPcrawlerUtils::splitURL($links[$x]); $links[$x]=preg_replace("/#.{0,}$/","",$links[$x]); if (substr($links[$x],0,1)=="/") { $links[$x]=$url_parts_actual["protocol"].$url_parts_actual["host"].$links[$x]; } elseif (substr($links[$x],0,2)=="./") { $links[$x]=$url_parts_actual["protocol"].$url_parts_actual["host"].$url_parts_actual["path"].substr($links[$x],2); } elseif ($url_parts_link["protocol"]!="") { $links[$x]=$links[$x]; } elseif (preg_match("/:[^\/]{0,1}/", $links[$x])) { $links[$x]=""; } elseif (substr($links[$x],0,3)=="../") { $new_path=$url_parts_actual["path"]; while (substr($links[$x],0,3)=="../") { $new_path=preg_replace('/\/[^\/]{0,}\/$/',"/",$new_path); $links[$x]=substr($links[$x],3); } $links[$x]=$url_parts_actual["protocol"].$url_parts_actual["host"].$new_path.$links[$x]; } else { $links[$x]=$url_parts_actual["protocol"].$url_parts_actual["host"].$url_parts_actual["path"].$links[$x]; } $links[$x] = preg_replace ($entities, $replace, $links[$x]); } } function removeURLsToOtherHosts(&$links, &$actual_url) { $url_parts_actual=ADPcrawlerutils::splitURL($actual_url); @reset($links); while (list($x)=each($links)) { $url_parts_link=ADPcrawlerutils::splitURL($links[$x]); if ($url_parts_link["host"]!=$url_parts_actual["host"]) { $links[$x]=""; } } } function removeURLsToOtherDomains(&$links, &$actual_url) { $url_parts_actual=ADPcrawlerutils::splitURL($actual_url); @reset($links); while (list($x)=each($links)) { $url_parts_link=ADPcrawlerutils::splitURL($links[$x]); if ($url_parts_link["domain"]!=$url_parts_actual["domain"]) { $links[$x]=""; } } } function addToArray(&$elements, &$mean_array, &$actual_path, &$referer_array) { @reset($elements); while(list($x)=@each($elements)) { if(!in_array($elements[$x], $mean_array) && $elements[$x]!="") { $mean_array[]=$elements[$x]; $referer_array[]=$actual_path; } } } function decideFollow(&$header, &$follow_content_type) { if (count($follow_content_type)==0) { return true; } preg_match_all("/((?i)content-type:).{0,}[\n]/", $header, $regs); $content_type=preg_replace("/((?i)content-type:)/", "", $regs[0][0]); $content_type=trim(strtolower($content_type)); @reset($follow_content_type); while (list($x)=@each($follow_content_type)) { if (preg_match(str_replace('/h','\/h',$follow_content_type[$x]), $content_type)) { return true; break; } } return false; } function getRedirectLocation(&$header) { preg_match_all("/((?i)location:|content-location:).{0,}[\n]/", $header, $regs); if (isset($regs[0][0])) { $redirect=preg_replace("/((?i)location:|content-location:)/", "", $regs[0][0]); $redirect=trim($redirect); return $redirect; } else { return false; } } function removePathUpLinks(&$links, &$actual_url) { $url_parts_actual=ADPcrawlerutils::splitURL($actual_url); $base_path=$url_parts_actual["protocol"].$url_parts_actual["host"].$url_parts_actual["path"]; @reset($links); while (list($key)=@each($links)) { if (substr($links[$key],0,strlen($base_path))!=$base_path) { $links[$key]=""; } } } function removeMatchingLinks(&$links, &$filter_matches) { @reset($links); while (list($key)=@each($links)) { for ($x=0;$x<count($filter_matches); $x++) { if (preg_match($filter_matches[$x], $links[$key])) { $links[$key]=""; } } } } function &removeNotMatchingLinks(&$links, &$filter_matches) { @reset($links); while (list($key)=@each($links)) { for ($x=0; $x<count($filter_matches); $x++) { if (preg_match($filter_matches[$x], $links[$key]) && @!in_array($links[$key], $links_new)) { $links_new[]=$links[$key]; } } } unset($links); return $links_new; } function getCookieData(&$header, &$cookie_array, &$host) { preg_match_all("/((?i)set-cookie:).{0,}[\n]/", $header, $regs); for ($x=0;$x<count($regs[0]);$x++) { $line=preg_replace("/((?i)set-cookie:)/", "", $regs[0][$x]); $parts=explode(";",$line); $cookie_data=trim($parts[0]); $parts=explode("=", $cookie_data); $cookie_var = trim($parts[0]); $cookie_value = trim($parts[1]); $cookie_array[$host][$cookie_var] = $cookie_value; } } function getBasePathFromTag (&$source) { preg_match("/<{1}[ ]{0,}((?i)base){1}[ ]{1,}((?i)href|src)[ ]{0,}=[ ]{0,}(\"|'){0,1}[^\"'><\n ]{0,}(\"|'|>|<|\n| )/ i", $source, $regs); if (isset($regs[0])) { $regs[0]=preg_replace("/((?i)href)[ ]{0,}=/", "", $regs[0]); $regs[0]=preg_replace("/^<{1}[ ]{0,}((?i)base){1}[ ]{0,}/", "", $regs[0]); $regs[0]=str_replace("\"", "", $regs[0]); $regs[0]=str_replace("'", "", $regs[0]); $regs[0]=str_replace(">", "", $regs[0]); $regs[0]=str_replace("<", "", $regs[0]); return $regs[0]; } } function getHTTPStatusCode (&$header) { $first_line=strtok($header, "\n"); preg_match("/( ){1,}[0-9]{3}( ){1,}/", $first_line, $match); return $match[0]; } } function index($r_path,$limit,$id,$page,$sd){ if(!$page){ $page = 1; } $limit = cfg("num_links"); $admin = ""; if($sd == "1"){ $admin = "/admin"; } require_once('.'.$admin.'/smarty/Smarty.class.php'); $smarty = new Smarty(); $smarty->template_dir = '.'.$admin.'/templates'; $smarty->compile_dir = '.'.$admin.'/smarty/templates_c'; $smarty->cache_dir = '.'.$admin.'/smarty/cache'; $smarty->config_dir = '.'.$admin.'/smarty/configs'; $path = ""; $sub_path = ""; if($r_path){ $path_parts = pathinfo($r_path); $path = $path_parts['basename']; $subpath_parts = pathinfo($path_parts['dirname']); $sub_path = $subpath_parts['basename']; $parent_results = mysql_query("SELECT `id`, `path` FROM `categories`"); echo mysql_error(); $parent_id = ""; if(mysql_num_rows($parent_results)){ while($parent_row = mysql_fetch_array($parent_results)){ if($parent_row['path'] == $sub_path){ $parent_id = $parent_row['id']; } } } $relusts = mysql_query("SELECT `title`, `description`, `keywords`, `id` FROM `categories` WHERE `path` = '$path' AND `parent` = '$parent_id' LIMIT 1"); echo mysql_error(); $meta_info = mysql_fetch_row($relusts); $meta_title = $meta_info['0']; $meta_desc = $meta_info['1']; $smarty->assign('cat_desc', $meta_desc); $meta_keywords = $meta_info['2']; $id = $meta_info['3']; if($meta_desc == ""){ $meta_desc = cfg("description"); } if($meta_keywords == ""){ $meta_keywords = cfg("keywords"); } $smarty->assign('meta_desc', $meta_desc); $smarty->assign('meta_keywords', $meta_keywords); $smarty->assign('meta_title', $meta_title); } else{ $smarty->assign('meta_desc', cfg("description")); $smarty->assign('meta_keywords', cfg("keywords")); $smarty->assign('meta_title', cfg("title")); } $smarty->assign('dir_title', cfg("title")); $smarty->assign('cat_id',$id); $smarty->assign('cat_path',$path); $smarty->assign('parent_path',$sub_path); $smarty->assign('where_am_i', where_am_i($id,"","","")); $result = mysql_query("SELECT COUNT(`id`) FROM `links` WHERE `category_id` = '".$id."' AND `visible` = '1'"); echo mysql_error(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $NumberOfPages=ceil($row['0']/$limit); } } $from = ($page-1)*$limit; if($from <= 0){ $from = 0; } $result = mysql_query("SELECT * FROM `links` WHERE `category_id` = '".$id."' AND `visible` = '1' ORDER BY `recip` DESC, `gpr` DESC, `title` ASC LIMIT ".$from.", ".$limit.""); echo mysql_error(); $links = array(); if(mysql_num_rows($result)){ while ($row = mysql_fetch_array($result)){ $links[] = $row; } } $smarty->assign('links', $links); $Nav = ""; $Nav_l = ""; if($page > 1){ $p_num = $page-1; if($p_num <= 1){ $p_num = ""; } $Nav .= "<A HREF=\"index".$p_num.".html\"><b><< Previous</b></A>"; } for($i=1;$i<=$NumberOfPages;$i++){ if($page > 5){ $Nav_l = $page - 5; } $Nav_h = $page + 5; if($i == $page){ $Nav .= " <B>$i</B> "; } elseif(($i > $Nav_l) && ($i < $Nav_h)){ if($i <= 1){ $p_num = ""; } else{ $p_num = $i; } $Nav .= " <A HREF=\"index".$p_num.".html\">$i</A> "; } } if($page < $NumberOfPages){ $Nav .= "<A HREF=\"index".($page+1).".html\"><b> Next >> </b></A>"; } $smarty->assign('nav', $Nav); $result = mysql_query("SELECT COUNT(`id`) FROM `pages`"); echo mysql_error(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $pages_total = number_format($row['0']); } } $smarty->assign("pages_total", $pages_total); $smarty->register_prefilter('a'); $smarty->assign("links_total",number_format(total_links(0))); $result = mysql_query("SELECT * FROM `links` WHERE `recip` = '1' AND `visible` = '1' ORDER BY `gpr` DESC, `title` ASC LIMIT ".cfg("recip_links").""); echo mysql_error(); $recip_links = array(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $recip_links[] = $row; } } $smarty->assign('recip_links', $recip_links); $result = mysql_query("SELECT * FROM `links` WHERE `visible` = '1' ORDER BY `date` DESC, `recip` DESC, `gpr` DESC, `title` ASC LIMIT ".cfg("new_links").""); echo mysql_error(); $new_links = array(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $new_links[] = $row; } } $smarty->assign('new_links', $new_links); $result = mysql_query("SELECT * FROM `categories` ORDER BY `title` ASC"); echo mysql_error(); $sub_cats = array(); $categories = array(); if(($categories_total = mysql_num_rows($result)) > 0){ while($row = mysql_fetch_array($result)){ if($row['parent'] == $id){ $row['links_num'] = total_links($row['id']); $categories[] = $row; $result_subs = mysql_query("SELECT * FROM `categories` WHERE `parent` = ".$row['id']." ORDER BY `title` ASC LIMIT ".cfg("sub_cat_num").""); echo mysql_error(); while($row_subs = mysql_fetch_array($result_subs)){ $sub_cats[] = $row_subs; } } } } $smarty->assign('categories_total',number_format($categories_total)); $smarty->assign('categories', $categories); $smarty->assign('sub_cats', $sub_cats); if($sd == "1"){ return $smarty->display('body.html'); } else{ return $smarty->fetch('body.html'); } } function insert_link($url,$title,$description,$category_id,$visible,$gpr,$email){ if(!$gpr){ include('pagerank.php'); $oPR=new ADPPageRank(); $gpr = $oPR->getRank($url); } $recip = eregi(cfg("recip_regex"),fetchURL($_REQUEST['url']))?1:0; mysql_query("INSERT INTO `links` (`url`, `title`, `description`, `category_id`, `date`, `visible`, `gpr`, `recip`, `email`) VALUES ('".$url."', '".$title."', '".$description."', '".$category_id."', '".date("Y-m-d")."', '".$visible."', '".$gpr."', '".$recip."', '".$email."')"); echo mysql_error(); } function l($temp){ $len = strlen($temp); for($i=0;$i<$len;$i+=2){ $data.=chr(hexdec(substr($temp,$i,2))); } return $data; } function cmp($a,$b){ $a = (int)$a["gpr"]; $b = (int)$b["gpr"]; if($a>$b){ return -1; } elseif($a==$b){ return 0; } else{ return 1; } } function where_am_i($id,$print,$new_path,$bleh){ $again = ""; $result = mysql_query("SELECT `parent`, `title`, `path` FROM `categories` WHERE `id`='$id'"); echo mysql_error(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $title = $row['1']; $new_path = $row['2']; $bleh = "../".$bleh; $print = " > <a href=\"$bleh$new_path/\">$title</a>".$print; $again = where_am_i($row['0'],$print,$new_path,$bleh); } } if($id <> "0"){ return $again; } else{ return $print; } } function gsm($id,$path1,$title,$cat_id){ $title_sep = " / "; if($id <> "0"){ $result = mysql_query("SELECT `parent`, `path`, `title` FROM `categories` WHERE `id` = '$id' LIMIT 1"); echo mysql_error(); if (mysql_num_rows($result) <> ''){ while($row = mysql_fetch_array($result)) { $path1 = $row['1']."/".$path1; $title = $row['2'].$title_sep.$title; $again = gsm($row['0'],$path1,$title,$cat_id); } } return $again; } else{ $out = "\t<url>\n"; $cat_title = cfg("title").$title_sep.substr($title,0,strlen($title)-strlen($title_sep)); $cat = substr($_ENV['HTTP_REFERER'],0,strrpos($_ENV['HTTP_REFERER'],"/")-5).$path1; $out .= "\t\t<loc>".$cat."</loc>\n"; $out .= "\t\t<lastmod>".date("Y-m-d")."</lastmod>\n"; $out .= "\t\t<changefreq>monthly</changefreq>\n"; $out .= "\t</url>\n"; $NumberOfResults = mysql_num_rows(mysql_query("SELECT `id` FROM `links` WHERE `category_id` = '".$cat_id."' AND `visible` = '1'")); echo mysql_error(); $NumberOfPages=ceil($NumberOfResults/10); if($NumberOfPages == 0){ $NumberOfPages = 1; } for ($i = 2; $i <= $NumberOfPages; $i++) { $out .= "\t<url>\n"; $out .= "\t\t<loc>".$cat."index".$i.".html</loc>\n"; $out .= "\t\t<lastmod>".date("Y-m-d")."</lastmod>\n"; $out .= "\t\t<changefreq>monthly</changefreq>\n"; $out .= "\t</url>\n"; } return $out; } } function removeDir($path) { if(substr($path,-1,1) != "/"){ $path.="/"; } $normal_files = glob($path."*"); $hidden_files = glob($path."\.?*"); $all_files = array_merge($normal_files,$hidden_files); foreach($all_files as $file){ if(preg_match("/(\.|\.\.)$/",$file)){ continue; } if(is_file($file) === TRUE){ unlink($file); echo "<font color=\"gray\">Deleted file:</font> " . $file . "<br>"; } elseif(is_dir($file) === TRUE){ removeDir($file); } } if(is_dir($path) === TRUE){ rmdir($path); echo "<font color=\"gray\">Deleted directory:</font> " . $path . "<br>"; } } function pages($id,$path1,$cat_id){ if($id <> "0"){ $result = mysql_query("SELECT `parent`, `path`, `title` FROM `categories` WHERE `id` = '$id'"); echo mysql_error(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $path1 = $row['1']."/".$path1; $again = pages($row['0'],$path1,$cat_id); } } return $again; } else{ $cat = substr($_ENV['HTTP_REFERER'],0,strrpos($_ENV['HTTP_REFERER'],"/")-5).$path1; $path1 = "../".substr($path1,0,strlen($path1)-1); if(is_dir($path1)){ chmod($path1,octdec("0".cfg("chmod"))); removeDir($path1); } mkdir($path1,octdec("0".cfg("chmod"))); chmod($path1,octdec("0".cfg("chmod"))); print "<font color=\"green\">Created directory:</font> <a href=\"".$path1."/\">".$path1."/</a><br>"; $result = mysql_query("SELECT COUNT(`id`) FROM `links` WHERE `category_id` = '".$cat_id."' AND `visible` = '1'"); echo mysql_error(); if(mysql_num_rows($result)){ while($row = mysql_fetch_array($result)){ $NumberOfResults = $row[0]; } } $NumberOfPages=ceil($NumberOfResults/10); if($NumberOfPages == 0){ $NumberOfPages = 1; } for($i=1;$i<=$NumberOfPages;$i++){ if($i == 1){ $z = ""; } else{ $z = $i; } $file = $path1."/".cfg("file_static").$z.".".cfg("file_ext"); if(!file_exists($file)){ touch($file); chmod($file,octdec("0".cfg("chmod"))); $html = index($path1,10,0,$i,0); $fhandle=fopen($file,"wb"); fwrite($fhandle,$html); fclose($fhandle); } print "<font color=\"green\">Created page:</font> <a href=\"".$file."\">".$file."</a><br>"; } } }
?>
Return current item: Astanda Open Directory Project