75 lines
2.7 KiB
PHP
75 lines
2.7 KiB
PHP
|
<?php
|
||
|
|
|
||
|
|
require_once('/home/cloud/core/api/php/db.php');
|
||
|
|
require_once('/home/cloud/core/api/php/json.php');
|
||
|
|
DB::$type='sqlite3';
|
||
|
|
DB::$path='/home/cloud/core/bd/tk-ligat.ru.db';
|
||
|
|
|
||
|
|
function leopak ($text) {
|
||
|
|
$old1 = array("https://tk-ligat.ru");
|
||
|
|
$new1 = array("");
|
||
|
|
$text = str_replace($old1, $new1, $text);
|
||
|
|
return $text;
|
||
|
|
}
|
||
|
|
|
||
|
|
function add_link($html){
|
||
|
|
preg_match_all("/<[Aa][\s]{1}[^>]*[Hh][Rr][Ee][Ff][^=]*=[ '\"\s]*([^ \"'>\s#]+)[^>]*>/", $html, $matches);
|
||
|
|
$urls = $matches[1]; // Берём то место, где сама ссылка (благодаря группирующим скобкам в регулярном выражении)
|
||
|
|
/* Выводим все ссылки */
|
||
|
|
for ($j = 0; $j < count($urls); $j++){
|
||
|
|
// sleep(1);
|
||
|
|
$link=leopak($urls[$j]);
|
||
|
|
// echo $link . "\n";
|
||
|
|
// echo "SELECT `id` FROM `sitemap` WHERE `link`=? LIMIT 1 \n";
|
||
|
|
$id=DB::getValue("SELECT `id` FROM `sitemap` WHERE `link`='" . $link . "' LIMIT 1");
|
||
|
|
if (!$id)DB::add("INSERT INTO `sitemap` (`link`, `t`, `status`) VALUES (?, ?, ?)", array($link, time(), 1));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
function get_page($link){
|
||
|
|
$agent = 'Mozilla/5.0 (compatible; YandexBot/3.0)';
|
||
|
|
// echo $link . "\n\r";
|
||
|
|
$ch = curl_init($link);
|
||
|
|
curl_setopt($ch, CURLOPT_REFERER, 'https://www.samsonopt.ru');
|
||
|
|
curl_setopt($ch, CURLOPT_USERAGENT, $agent);
|
||
|
|
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
|
||
|
|
$dir = dirname(__FILE__);
|
||
|
|
$config['cookie_file'] = $dir . '/cookies/' . md5(@$_SERVER['REMOTE_ADDR']) . '.txt';
|
||
|
|
curl_setopt($ch, CURLOPT_COOKIEFILE, $config['cookie_file']);
|
||
|
|
curl_setopt($ch, CURLOPT_COOKIEJAR, $config['cookie_file']);
|
||
|
|
curl_setopt($ch, CURLOPT_COOKIE, "PMBC=96152e8e9a0168a731539c5e52c6b39a; PHPSESSID=jl0i13pn3157qca807jgp0jqa7; ServerName=WoW+Circle+3.3.5a+x5; serverId=1");
|
||
|
|
// curl_setopt($ch, CURLOPT_PROXY, 'localhost:9050'); // Use if proxy have username and password
|
||
|
|
// curl_setopt($ch, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
|
||
|
|
$response_data = curl_exec($ch);
|
||
|
|
if (curl_errno($ch) > 0) {
|
||
|
|
die('Ошибка curl: ' . curl_error($ch));
|
||
|
|
}
|
||
|
|
curl_close($ch);
|
||
|
|
return $response_data;
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
$LinkList = DB::getAll( 'SELECT * FROM `sitemap` WHERE `status`=1 AND `check`=0 ORDER BY RANDOM() LIMIT 1000');
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
for ($i=0; $i<count( $LinkList ); $i++){
|
||
|
|
// sleep(1);
|
||
|
|
$link='https://tk-ligat.ru' . $LinkList[$i]['link'];
|
||
|
|
|
||
|
|
//$current=file_get_contents('../goods_sur102215_dcc93d1672486819.xml');
|
||
|
|
$html=get_page ( $link );
|
||
|
|
if ($html){
|
||
|
|
add_link($html);
|
||
|
|
}
|
||
|
|
DB::set("UPDATE `sitemap` SET `check`=1 WHERE `link`=?", $LinkList[$i]['link'] );
|
||
|
|
|
||
|
|
#exec ("php robot.php");
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
?>
|