Your IP : 216.73.216.218
<?php
/*
* Copyright (c) 2017-2019 Aimy Extensions, Netzum Sorglos Software GmbH
* Copyright (c) 2014-2017 Aimy Extensions, Lingua-Systems Software GmbH
*
* https://www.aimy-extensions.com/
*
* License: GNU GPLv2, see LICENSE.txt within distribution and/or
* https://www.aimy-extensions.com/software-license.html
*/
defined( '_JEXEC' ) or die(); jimport( 'joomla.filesystem.file' ); class AimySitemapRobotsTxt { private $rules = null; private $txt = null; const ALLOW = true; const DISALLOW = false; public function __construct( $txt = null ) { $this->rules = array(); if ( is_null( $txt ) ) { $this->txt = self::get_from_file(); if ( is_null( $this->txt ) ) { $this->txt = self::get_via_http(); if ( ! is_null( $this->txt ) ) { AimySitemapLogger::debug( 'RobotsTxt: Got robots.txt via HTTP' ); } } else { AimySitemapLogger::debug( 'RobotsTxt: Got robots.txt from file system' ); } if ( ! is_null( $this->txt ) ) { AimySitemapLogger::debug( 'RobotsTxt: ' . preg_replace( '#\r?\n#', '#NL#', $this->txt ) ); } } else { $this->txt = $txt; } if ( is_null( $this->txt ) ) { $this->txt = ''; } $this->rules = self::parse( $this->txt ); } public function get_txt() { return $this->txt; } public function disallowed( $url ) { if ( empty( $this->rules ) ) { return false; } foreach ( $this->rules as $r ) { if ( isset( $r[ 'regex' ] ) ) { if ( preg_match( $r[ 'regex' ], $url ) ) { return ( $r[ 'action' ] == self::DISALLOW ); } } elseif ( strpos( $url, $r[ 'path' ] ) === 0 ) { return ( $r[ 'action' ] == self::DISALLOW ); } } return false; } public function allowed( $url ) { return (! $this->disallowed( $url )); } static private function get_from_file() { if ( JURI::root( true ) !== '/' ) { return null; } $path = JPATH_ROOT . '/robots.txt'; if ( JFile::exists( $path ) ) { return file_get_contents( $path ); } return null; } static private function get_via_http() { require_once( JPATH_ADMINISTRATOR . '/components/com_aimysitemap/HttpClient.php' ); try { $u = new AimySitemapURI( JURI::root() ); $u->setPath( '/robots.txt' ); $r = AimySitemapHttpClient::get_url( $u ); if ( is_array( $r ) && $r[ 'head' ][ 'code' ] == 200 ) { return $r[ 'body' ]; } } catch ( Exception $e ) { } return null; } static private function sort_desc_by_length( $a, $b ) { return strlen( $b[ 'path' ] ) - strlen( $a[ 'path' ] ); } static private function parse( $txt ) { $ls = preg_split( '/\r?\n/', $txt ); $apply = false; $rules = array(); foreach ( $ls as $l ) { $l = trim( $l ); if ( empty( $l ) or strpos( $l, '#' ) === 0 ) { continue; } if ( strpos( $l, ':' ) === false ) { throw new RuntimeException( 'Invalid robots.txt entry found: [' . $l . ']' ); } list( $op, $val ) = explode( ':', $l, 2 ); switch ( $op ) { case 'User-agent': case 'User-Agent': case 'user-agent': $apply = (( strpos( $val, '*' ) !== false ) or ( strpos( $val, 'AimySitemapCrawler' ) !== false )); break; case 'Disallow': case 'disallow': if ( $apply ) { $val = trim( $val ); if ( empty( $val ) ) { $rules = array(); } else { $rule = array( 'path' => $val, 'action' => self::DISALLOW ); if ( self::path_uses_pattern( $val ) ) { $rule[ 'regex' ] = self::pattern_to_regex( $val ); } $rules[] = $rule; } } break; case 'Allow': case 'allow': if ( $apply ) { $val = trim( $val ); if ( ! empty( $val ) ) { $rule = array( 'path' => $val, 'action' => self::ALLOW ); if ( self::path_uses_pattern( $val ) ) { $rule[ 'regex' ] = self::pattern_to_regex( $val ); } $rules[] = $rule; } } break; case 'Crawl-delay': case 'Crawl-Delay': case 'crawl-delay': case 'Sitemap': case 'sitemap': case 'Host': case 'host': case 'Noindex': case 'noindex': break; default: throw new RuntimeException( 'Invalid robots.txt entry found: [' . $l . ']' ); } } usort( $rules, 'self::sort_desc_by_length' ); return $rules; } static private function pattern_to_regex( $path ) { $end = false; $len = strlen( $path ); if ( $len && substr( $path, $len - 1, 1 ) === '$' ) { $path = substr( $path, 0, $len - 1 ); $end = true; } $rep = explode( '*', $path ); foreach ( $rep as $i => $p ) { if ( ! empty( $p ) ) { $rep[ $i ] = '\Q' . $p . '\E'; } } $re = '#' . '^' . implode( '.*?', $rep ) . ( $end ? '$' : '' ) . '#'; return $re; } static private function path_uses_pattern( $path ) { return ( strpos( $path, '*' ) !== false or strpos( $path, '$' ) !== false ); } }