Your IP : 216.73.216.218
<?php
/*
* Copyright (c) 2017-2019 Aimy Extensions, Netzum Sorglos Software GmbH
* Copyright (c) 2014-2017 Aimy Extensions, Lingua-Systems Software GmbH
*
* https://www.aimy-extensions.com/
*
* License: GNU GPLv2, see LICENSE.txt within distribution and/or
* https://www.aimy-extensions.com/software-license.html
*/
defined( '_JEXEC' ) or die(); require_once( JPATH_ADMINISTRATOR . '/components/com_aimysitemap/HttpClient.php' ); require_once( JPATH_ADMINISTRATOR . '/components/com_aimysitemap/RobotsTxt.php' ); require_once( JPATH_ADMINISTRATOR . '/components/com_aimysitemap/Uri.php' ); require_once( JPATH_ADMINISTRATOR . '/components/com_aimysitemap/helpers/config.php' ); require_once( JPATH_ADMINISTRATOR . '/components/com_aimysitemap/helpers/kvstore.php' ); class AimySitemapCrawler { private $base = null; private $robtxt = null; private $dbh = null; private $cfg = null; private $curdir = null; const URL_MAX_LENGTH = 767; const MSG_NON_200 = 1; const MSG_ROBOTS_META = 2; const MSG_EXCLUDE_PATTERN = 4; const MSG_ROBOTS_TXT = 8; public function __construct() { $this->base = new AimySitemapURI( JURI::root() ); $this->dbh = JFactory::getDbo(); $this->cfg = new AimySitemapConfigHelper(); } public function initialize() { AimySitemapLogger::debug( 'Crawl-Init: Aimy Sitemap v24.3, base: ' . $this->base->toString() ); try { self::delete_crawl_data(); } catch ( Exception $e ) { return array( 'ok' => false, 'msg' => $e->getMessage() ); } if ( $this->cfg->get( 'crawl_disguise' ) ) { AimySitemapLogger::debug( 'Crawl-Init: Disguise mode enabled' ); AimySitemapHttpClient::set_disguise(); } try { $this->robtxt = new AimySitemapRobotsTxt(); } catch ( Exception $e ) { return array( 'ok' => false, 'msg' => $e->getMessage() ); } if ( $this->robtxt->disallowed( $this->base->getResource() ) ) { AimySitemapLogger::debug( 'Crawl-Init: Base disallowed by robots.txt' ); return array( 'abort' => true, 'ok' => false, 'msg' => self::code_to_msg( self::MSG_ROBOTS_TXT, $this->base->getResource() ) ); } try { AimySitemapKVStore::set( 'robots.txt', $this->robtxt->get_txt() ); } catch ( Exception $e ) { return array( 'ok' => false, 'msg' => $e->getMessage() ); } $this->curdir = self::get_curdir( $this->base->getResource() ); $this->handle_found_link( $this->base ); return $this->crawl(); } public function crawl() { try { $this->load_robotstxt_object(); } catch ( Exception $e ) { return array( 'abort' => true, 'ok' => false, 'msg' => $e->getMessage() ); } $url = null; try { $url = $this->get_next_url(); } catch ( Exception $e ) { return array( 'abort' => true, 'ok' => false, 'msg' => $e->getMessage() ); } if ( empty( $url ) ) { return array( 'abort' => true, 'ok' => true ); } $u = new AimySitemapURI( $url ); AimySitemapLogger::debug( 'Crawl: next: ' . $u->getResource() ); $this->curdir = self::get_curdir( $u->getResource() ); usleep( $this->cfg->get( 'crawl_sleep' ) * 1000000 ); $obj = new StdClass(); $obj->url = $u->getResource(); $obj->crawled = 1; $obj->index = 0; $obj->title = ''; $obj->mtime = 0; $obj->lang = '*'; $obj->code = 0; $obj->refs = ''; $r = false; try { $u->setPort( $this->base->getPort() ); $u->setHost( $this->base->getHost() ); $u->setScheme( 'http' ); if ( $this->cfg->get( 'crawl_disguise' ) ) { AimySitemapHttpClient::set_disguise(); } $r = AimySitemapHttpClient::head_url( $u ); } catch ( Exception $e ) { return array( 'ok' => false, 'msg' => self::code_to_msg( $e->getCode(), $e->getMessage() ), 'url' => $obj->url ); } try { AimySitemapKVStore::set( 'crawl-protocol', $u->getScheme() ); } catch ( Exception $e ){ } if ( ( $r[ 'head' ][ 'code' ] == '500' or $r[ 'head' ][ 'code' ] == '403' or $r[ 'head' ][ 'code' ] == '404' ) && strpos( $obj->url, JUri::root( true ) . '/images/' ) !== 0 ) { AimySitemapLogger::debug( 'Crawl: HEAD/GET workaround used ' . '(' . $r[ 'head' ][ 'code' ] . ')' ); try { $r2 = AimySitemapHttpClient::get_url( $u ); if ( is_array( $r2 ) && $r2[ 'head' ][ 'code' ] == '200' ) { $r = $r2; unset( $r2 ); } } catch ( Exception $e ) { } } $obj->code = intVal( $r[ 'head' ][ 'code' ] ); if ( $obj->code == 200 ) { if ( isset( $r[ 'head' ][ 'Content-Type' ] ) and strpos( $r[ 'head' ][ 'Content-Type' ], 'html' ) > 0 ) { if ( ! isset( $r[ 'body' ] ) ) { try { $r = AimySitemapHttpClient::get_url( $u ); } catch ( Exception $e ) { return array( 'ok' => false, 'msg' => self::code_to_msg( $e->getCode(), $e->getMessage() ), 'url' => $obj->url ); } } $is_html_sm = false; if ( $is_html_sm = self::has_html_sitemap_id( $r[ 'body' ] ) ) { $r[ 'body' ] = self::strip_html_sitemap( $r[ 'body' ] ); } $meta_robots = self::extract_meta_robots( $r[ 'body' ] ); if ( strpos( $meta_robots, 'nofollow' ) === false ) { $this->extract_links( $r[ 'body' ] ); } else { AimySitemapLogger::debug( 'Crawl: meta: nofollow set' ); } if ( $is_html_sm ) { return array( 'ok' => true, 'msg' => 'Aimy Sitemap HTML Sitemap', 'url' => $obj->url ); } if ( strpos( $meta_robots, 'noindex' ) !== false ) { return array( 'ok' => true, 'msg' => self::code_to_msg( self::MSG_ROBOTS_META ), 'url' => $obj->url ); } $obj->title = self::extract_title( $r[ 'body' ] ); } $expats = $this->cfg->get_splitted( 'crawl_exclude_patterns' ); foreach ( $expats as $expat ) { if ( self::match_pattern( $expat, $obj->url ) ) { return array( 'ok' => true, 'msg' => self::code_to_msg( self::MSG_EXCLUDE_PATTERN, $expat ), 'url' => $obj->url ); } } if ( isset( $r[ 'head' ][ 'Last-Modified' ] ) ) { $ts = strtotime( $r[ 'head' ][ 'Last-Modified' ] ); if ( $ts !== false ) { $obj->mtime = $ts; } } if ( empty( $obj->title ) ) { $obj->title = basename( $u->getResourceHTML() ); } $obj->index = 1; try { $this->update_url( $obj ); } catch ( Exception $e ) { return array( 'ok' => false, 'msg' => $e->getMessage(), 'url' => $obj->url ); } return array( 'ok' => true, 'url' => $obj->url ); } else if ( $obj->code == 301 or $obj->code == 302 or $obj->code == 303 or $obj->code == 307 or $obj->code == 308 ) { if ( ! isset( $r[ 'head' ][ 'Location' ] ) ) { return array( 'ok' => false, 'msg' => 'Redirect without assigned Location', 'url' => $obj->url ); } else { $this->handle_found_link( $r[ 'head' ][ 'Location' ] ); return array( 'ok' => true, 'msg' => $r[ 'head' ][ 'code' ] . ' → ' . $r[ 'head' ][ 'Location' ], 'url' => $obj->url ); } } else { return array( 'ok' => false, 'msg' => self::code_to_msg( self::MSG_NON_200, $r[ 'head' ][ 'code' ] ), 'url' => $obj->url ); } } private function extract_links( &$body ) { $links = array(); if ( preg_match_all( '#<a\s+([^>]+)>#i', $body, $ms, PREG_SET_ORDER ) ) { foreach ( $ms as $i => $m ) { if ( preg_match( '#rel=["\']?([^"\']+)["\']?#i', $m[1], $mr ) ) { if ( stripos( $mr[1], 'nofollow' ) !== false ) { continue; } } $l = self::get_attrval( $m[1], 'href' ); if ( ! empty( $l ) ) { $links[ $l ] = 1; } } } if ( preg_match_all( '#<link\s+([^>]+)>#i', $body, $ms, PREG_SET_ORDER ) ) { foreach ( $ms as $i => $m ) { if ( preg_match( '#rel=["\']?alternate["\']?#i', $m[1] ) && stripos( $m[1], 'hreflang=' ) !== false ) { $l = self::get_attrval( $m[1], 'href' ); if ( ! empty( $l ) ) { $links[ $l ] = 1; } } } } if ( $this->cfg->get( 'crawl_img', false ) ) { if ( preg_match_all( '#<img\s+([^>]+)>#i', $body, $ms, PREG_SET_ORDER ) ) { foreach ( $ms as $i => $m ) { $l = self::get_attrval( $m[1], 'src' ); if ( ! empty( $l ) ) { $links[ $l ] = 1; } } } } foreach ( array_keys( $links ) as $l ) { $this->handle_found_link( $l ); } } static private function extract_title( &$body ) { if ( preg_match( '#<title>(.*?)</title>#si', $body, $m ) ) { return str_replace( "\n", ' ', str_replace( "\r\n", ' ', trim( $m[1] ) ) ); } return ''; } static private function extract_meta_robots( &$body ) { if ( preg_match( '#<meta[^>]+name=["\']robots["\'][^>]*>#i', $body, $m ) ) { $c = self::get_attrval( $m[0], 'content' ); if ( ! empty( $c ) ) { return $c; } } return ''; } static private function has_html_sitemap_id( &$body ) { return (( strpos( $body, 'aimysitemap-list' ) !== false ) or ( strpos( $body, 'aimysitemap-index' ) !== false ) or ( strpos( $body, 'aimysitemap-hierarchy' ) !== false ) or ( strpos( $body, 'aimysitemap-priority' ) !== false ) ); } static private function strip_html_sitemap( &$body ) { return preg_replace( '#<div\s+id=["\']?aimysitemap-' . '(?:list|index|hierarchy|priority)["\']?>' . '.*?' . '</div><!-- /\#aimysitemap-\w+ -->#s', '', $body ); } static private function get_curdir( $p ) { return preg_replace( '#/{2,}#', '/', preg_replace( '#/[^/]+(?:\?.*)?$#', '/', $p ) ); } private function load_robotstxt_object() { $this->robtxt = new AimySitemapRobotsTxt( AimySitemapKVStore::get( 'robots.txt' ) ); } private function handle_found_link( $l ) { if ( empty( $l ) ) { return; } if ( strpos( $l, '/' ) !== 0 ) { if ( stripos( $l, 'mailto:' ) === 0 or stripos( $l, 'tel:' ) === 0 or stripos( $l, 'data:' ) === 0 or stripos( $l, 'skype:' ) === 0 or stripos( $l, 'whatsapp:' ) === 0 or stripos( $l, 'javascript:' ) === 0 or strpos( $l, '#' ) === 0 ) { return; } if ( strpos( $l, 'http' ) !== 0 ) { $l = $this->curdir . $l; } } if ( strlen( $l ) > self::URL_MAX_LENGTH ) { return; } if ( strpos( $l, '//' ) === 0 ) { $l = $this->base->getScheme() . ':' . $l; } $u = new AimySitemapURI( $l ); if ( $u->getHost() && $u->getHost() !== $this->base->getHost() ) { return; } $rsrc = $u->getResource(); if ( strlen( $rsrc ) > self::URL_MAX_LENGTH ) { AimySitemapLogger::debug( 'Crawl: WARN: URL exceeds maximum length of ' . self::URL_MAX_LENGTH . ' characters: ' . $rsrc ); return; } if ( strpos( $rsrc, $this->base->getResource() ) !== 0 ) { return; } if ( strpos( $u->getPath(), $this->base->getPath() . 'administrator/' ) === 0 ) { return; } if ( strpos( $rsrc, '//' ) === 0 ) { $rsrc = preg_replace( '#^/+#', '/', $rsrc ); } if ( $this->robtxt->disallowed( $rsrc ) ) { return; } try { $this->dbh->transactionStart(); $q = $this->dbh->getQuery( true ); $q->select( $this->dbh->quoteName( 'url' ) ) ->from( $this->dbh->quoteName( '#__aimysitemap_crawl' ) ) ->where( $this->dbh->quoteName( 'url' ) . ' = ' . $this->dbh->quote( $rsrc ) ) ->setLimit( 1 ); $this->dbh->setQuery( $q ); $row = $this->dbh->loadRow(); if ( ! empty( $row ) ) { $this->dbh->transactionCommit(); return; } $obj = new StdClass(); $obj->url = $rsrc; $obj->crawled = 0; $obj->index = 0; $obj->title = ''; $obj->mtime = 0; $obj->code = 0; $obj->refs = ''; AimySitemapLogger::debug( 'Crawl: +url: ' . $obj->url ); $this->dbh->insertObject( '#__aimysitemap_crawl', $obj ); $this->dbh->transactionCommit(); } catch ( Exception $e ) { $this->dbh->transactionRollback(); throw new Exception( $e->getMessage() ); } } private function update_url( $obj ) { $c = $this->dbh->updateObject( '#__aimysitemap_crawl', $obj, 'url' ); if ( ! $c && ! $this->dbh->getAffectedRows() ) { throw new RuntimeException( 'Failed to insert updated URL object to the database' ); } } private function get_next_url() { $url = null; try { $this->dbh->transactionStart(); $q = $this->dbh->getQuery( true ); $q->select( $this->dbh->quoteName( array( 'url', 'crawled' ) ) ) ->from( '#__aimysitemap_crawl' ) ->where( $this->dbh->quoteName( 'crawled' ) . ' = 0' ) ->setLimit( 1 ); $this->dbh->setQuery( $q ); $robj = $this->dbh->loadObject(); if ( is_object( $robj ) and isset( $robj->url ) ) { $robj->crawled = 1; $this->dbh->updateObject( '#__aimysitemap_crawl', $robj, 'url' ); $url = $robj->url; } $this->dbh->transactionCommit(); } catch ( Exception $e ) { $this->dbh->transactionRollback(); throw new Exception( $e->getMessage() ); } return $url; } static public function delete_crawl_data() { AimySitemapKVStore::delete( 'robots.txt' ); return JFactory::getDbo()->truncateTable( '#__aimysitemap_crawl' ); } static private function code_to_msg( $code, $append = null ) { $msg = ''; switch ( $code ) { case AimySitemapHttpClient::MSG_PARSE_HTTP: case AimySitemapHttpClient::MSG_SOCKET: $msg = JText::sprintf( 'AIMY_SM_ERR_CONNECT', $append ? $append : '' ) . ' (' . $code . ')'; $append = null; break; case self::MSG_NON_200: $msg = 'HTTP Status Code' . ( $append ? '' : ': != 200' ); break; case self::MSG_ROBOTS_META: $msg = JText::_( 'AIMY_SM_MSG_ROBOTS_META' ); break; case self::MSG_EXCLUDE_PATTERN: $msg = JText::_( 'AIMY_SM_MSG_EXCLUDE_PATTERN' ); break; case self::MSG_ROBOTS_TXT: $msg = JText::_( 'AIMY_SM_MSG_ROBOTS_TXT' ); break; default: $msg = JText::_( 'AIMY_SM_MSG_ERRORS' ); break; } if ( ! is_null( $append ) ) { $msg .= ': ' . $append; } return $msg; } static private function match_pattern( $pat, $s ) { $pps = explode( '*', $pat ); foreach ( $pps as $i => $p ) { $pps[ $i ] = preg_quote( $p, '/' ); } $re = '/^' . implode( '.*?', $pps ) . '$/'; return ( preg_match( $re, $s ) === 1 ); } static private function get_attrval( &$s, $attr ) { $v = null; if ( preg_match( '#\Q' . $attr . '\E=([\"\'])([^\"\'>]*)\1#i', $s, $m ) ) { $v = $m[ 2 ]; } elseif ( preg_match( '#\Q' . $attr . '\E=([^\s>]*)#i', $s, $m ) ) { $v = $m[ 1 ]; } if ( ! empty( $v ) ) { $v = trim( $v, " \t\n\r\0\x0B'\"\\" ); } return $v; } }